Sequence in LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ch `10`: Concept `03`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recurrent Neural Network on real data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "import data_loader\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class SeriesPredictor:\n",
    "\n",
    "    def __init__(self, input_dim, seq_size, hidden_dim):\n",
    "        # Hyperparameters\n",
    "        self.input_dim = input_dim\n",
    "        self.seq_size = seq_size\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        # Weight variables and input placeholders\n",
    "        self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')\n",
    "        self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')\n",
    "        self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])\n",
    "        self.y = tf.placeholder(tf.float32, [None, seq_size])\n",
    "\n",
    "        # Cost optimizer\n",
    "        self.cost = tf.reduce_mean(tf.square(self.model() - self.y))\n",
    "        self.train_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(self.cost)\n",
    "\n",
    "        # Auxiliary ops\n",
    "        self.saver = tf.train.Saver()\n",
    "\n",
    "    def model(self):\n",
    "        \"\"\"\n",
    "        :param x: inputs of size [T, batch_size, input_size]\n",
    "        :param W: matrix of fully-connected output layer weights\n",
    "        :param b: vector of fully-connected output layer biases\n",
    "        \"\"\"\n",
    "        cell = rnn.BasicLSTMCell(self.hidden_dim)\n",
    "        outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)\n",
    "        num_examples = tf.shape(self.x)[0]\n",
    "        W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])\n",
    "        out = tf.matmul(outputs, W_repeated) + self.b_out\n",
    "        out = tf.squeeze(out)\n",
    "        return out\n",
    "\n",
    "    def train(self, train_x, train_y, test_x, test_y):\n",
    "        with tf.Session() as sess:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            max_patience = 3\n",
    "            patience = max_patience\n",
    "            min_test_err = float('inf')\n",
    "            step = 0\n",
    "            while patience > 0:\n",
    "                _, train_err = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})\n",
    "                if step % 100 == 0:\n",
    "                    test_err = sess.run(self.cost, feed_dict={self.x: test_x, self.y: test_y})\n",
    "                    print('step: {}\\t\\ttrain err: {}\\t\\ttest err: {}'.format(step, train_err, test_err))\n",
    "                    if test_err < min_test_err:\n",
    "                        min_test_err = test_err\n",
    "                        patience = max_patience\n",
    "                    else:\n",
    "                        patience -= 1\n",
    "                step += 1\n",
    "            save_path = self.saver.save(sess, 'model.ckpt')\n",
    "            print('Model saved to {}'.format(save_path))\n",
    "\n",
    "    def test(self, sess, test_x):\n",
    "        tf.get_variable_scope().reuse_variables()\n",
    "        self.saver.restore(sess, './model.ckpt')\n",
    "        output = sess.run(self.model(), feed_dict={self.x: test_x})\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_results(train_x, predictions, actual, filename):\n",
    "    plt.figure()\n",
    "    num_train = len(train_x)\n",
    "    plt.plot(list(range(num_train)), train_x, color='b', label='training data')\n",
    "    plt.plot(list(range(num_train, num_train + len(predictions))), predictions, color='r', label='predicted')\n",
    "    plt.plot(list(range(num_train, num_train + len(actual))), actual, color='g', label='test data')\n",
    "    plt.legend()\n",
    "    if filename is not None:\n",
    "        plt.savefig(filename)\n",
    "    else:\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-1-ef39e0a7efa3>:35: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-1-ef39e0a7efa3>:36: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "step: 0\t\ttrain err: 0.46601834893226624\t\ttest err: 2.981935501098633\n",
      "step: 100\t\ttrain err: 0.04192028567194939\t\ttest err: 0.23095989227294922\n",
      "step: 200\t\ttrain err: 0.03976554796099663\t\ttest err: 0.2402084916830063\n",
      "step: 300\t\ttrain err: 0.03768425062298775\t\ttest err: 0.2121000587940216\n",
      "step: 400\t\ttrain err: 0.03580646216869354\t\ttest err: 0.21552744507789612\n",
      "step: 500\t\ttrain err: 0.03398025780916214\t\ttest err: 0.22716522216796875\n",
      "step: 600\t\ttrain err: 0.034817975014448166\t\ttest err: 0.2574256658554077\n",
      "Model saved to model.ckpt\n",
      "WARNING:tensorflow:From C:\\Users\\User\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\training\\saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from ./model.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXl4VFXS/78nO1lIQhJCAtkQkDWETYjACIiIgjj64r6gjuAyOvoyIqKvwKD8ZhwV3EcREMdBREHUERBEiMguCMgOgSSQhSRkJft2fn9U316SXm5336S7Q32eJ0933+Xc4oZ8b3WdOlVCSgmGYRim/eDlagMYhmEYbWFhZxiGaWewsDMMw7QzWNgZhmHaGSzsDMMw7QwWdoZhmHYGCzvDMEw7g4WdYRimncHCzjAM087wcXYAIUQAgO0A/HXjrZFSzrN2TmRkpExMTHT20gzDMFcUBw4cuCSljLJ1nNPCDqAWwDgpZYUQwhfADiHERinlHksnJCYmYv/+/RpcmmEY5spBCJGl5jinhV1SsZkK3Udf3Q8XoGEYhnERmsTYhRDeQohDAAoA/Cil3KvFuAzDMIz9aCLsUspGKWUKgG4ArhFC9G9+jBBihhBivxBif2FhoRaXZRiGYcygRYxdj5SyVAiRBmAigKPN9i0BsAQAhg4d2iJUU19fj+zsbNTU1GhpEuMgAQEB6NatG3x9fV1tCsMwdqJFVkwUgHqdqHcAMB7Aa/aOk52djZCQECQmJkII4axZjBNIKVFUVITs7GwkJSW52hyGYexEi1BMDIBtQojfAfwKirF/b+8gNTU1iIiIYFF3A4QQiIiI4G9PDOOhaJEV8zuAQRrYwqLuRvDvgmE8F155yjAM4wQ/nfsJRwuO2j6wDWFh11FaWooPPvjAoXNvvvlmlJaWWj1m7ty52LJli0PjW2PFihV46qmnrB6TlpaGXbt2aX5thmGAR//7KGb8d4arzTCBhV2HNWFvbGy0eu6GDRsQFhZm9ZgFCxZg/PjxDtvnDCzsDNN6FFUVYXf2bmSWZrraFD0s7DpeeOEFnD17FikpKZg1axbS0tIwduxY3HvvvRgwYAAA4I9//COGDBmCfv36YcmSJfpzExMTcenSJWRmZqJPnz6YPn06+vXrhwkTJqC6uhoA8NBDD2HNmjX64+fNm4fBgwdjwIABOHnyJACgsLAQN9xwAwYPHozHHnsMCQkJuHTpUgtbP/nkE/Tq1QvXXXcddu7cqd/+3//+F8OHD8egQYMwfvx45OfnIzMzEx9++CEWL16MlJQU/PLLL2aPYxjGfhqaGnC57jIA4IujX7jYGgOa5rFrxbPPAocOaTtmSgrw1luW9//jH//A0aNHcUh34bS0NOzbtw9Hjx7Vp/wtX74cnTp1QnV1NYYNG4b/+Z//QUREhMk4Z86cwapVq/Dxxx/jzjvvxNq1a3H//fe3uF5kZCR+++03fPDBB3jjjTewdOlS/O1vf8O4ceMwZ84c/PDDDyYPD4W8vDzMmzcPBw4cQGhoKMaOHYtBg2juetSoUdizZw+EEFi6dCn++c9/4s0338Tjjz+O4OBgPPfccwCAkpISs8cxDGMfZTVl+vefH/kcL4x6wYXWGHBLYXcXrrnmGpM87nfeeQfr1q0DAFy4cAFnzpxpIexJSUlISUkBAAwZMgSZmZlmx7799tv1x3z99dcAgB07dujHnzhxIsLDw1uct3fvXowZMwZRUVTg7a677sLp06cB0FqAu+66C3l5eairq7OYg672OIZhrFNaQ3Nrw2KH4dfcX3Gs4Bj6de7nYqvcVNitedZtSVBQkP59WloatmzZgt27dyMwMBBjxowxm+ft7++vf+/t7a0PxVg6ztvbGw0NDQBoYZAaLKUiPv3005g5cyamTJmCtLQ0zJ8/36njGIaxTklNCQBg+uDpOJB3AKuOrsKr4151sVUcY9cTEhKCy5cvW9xfVlaG8PBwBAYG4uTJk9izx2JVYocZNWoUvvzySwDA5s2bUVJS0uKY4cOHIy0tDUVFRaivr8dXX31lYmPXrl0BAJ9++ql+e/N/m6XjGIaxD8Vj7x3ZG6ndUrEtc5uLLSJY2HVERERg5MiR6N+/P2bNmtVi/8SJE9HQ0IDk5GS8/PLLGDFihOY2zJs3D5s3b8bgwYOxceNGxMTEICQkxOSYmJgYzJ8/H6mpqRg/fjwGDx6s3zd//nzccccdGD16NCIjI/Xbb7nlFqxbt04/eWrpOIZh7KOkmpyvsIAwdAnuohd6VyPUfv3XkqFDh8rmjTZOnDiBPn36tLkt7kRtbS28vb3h4+OD3bt344knntBP5roC/p0wjHU+PvAxZnw/A+efPY95afOw+exmZM/MbrXrCSEOSCmH2jrOLWPsVyrnz5/HnXfeiaamJvj5+eHjjz92tUkMw1hB8dDDO4Qj1D8U5bXlLraIYGF3I3r27ImDBw+62gyGYVRSUlMCb+GNIN8ghAaE4nLdZTQ2NcLby9uldnGMnWEYxkFKa0oR3iEcQgh09O8IAPoFS66EhZ1hGMZBSmpKEBZA5URC/UMBwC3CMSzsDMMwDlJaU4rwAFpIqHjsxqtRXQULO8MwjIOUVBt57AHssV8RBAcHAwByc3MxdepUq8e+9dZbqKqqsmv8tLQ0TJ482WH7GIZxjtKaUr2w6z32WvbYPQ5bJXzNERsbq6/saAlHhJ1hGNdiHIrhGLubkpmZid69e2PatGlITk7G1KlTUVVVhcTERCxYsACjRo3CV199hbNnz2LixIkYMmQIRo8erS+7m5GRgdTUVAwbNgwvv/yyybj9+/cHQA+G5557DgMGDEBycjLeffddvPPOO8jNzcXYsWMxduxYAFRSIDU1FYMHD8Ydd9yBiooKAMAPP/yA3r17Y9SoUfriYQzDtD1SStPJU10oxh1i7O6Zx+6Kur06Tp06hWXLlmHkyJF45JFH9M03AgICsGPHDgDA9ddfjw8//BA9e/bE3r178eSTT2Lr1q145pln8MQTT+DBBx/E+++/b3b8JUuWICMjAwcPHoSPjw+Ki4vRqVMnLFq0CNu2bUNkZCQuXbqEV199FVu2bEFQUBBee+01LFq0CM8//zymT5+OrVu3okePHrjrrru0uz8Mw9hFTUMN6hrrEN6h2eSpG4Ri3FPYXUhcXBxGjhwJALj//vvxzjvvAIBeRCsqKrBr1y7ccccd+nNqa2sBADt37sTatWsBAA888ABmz57dYvwtW7bg8ccfh48P3fpOnTq1OGbPnj04fvy43o66ujqkpqbi5MmTSEpKQs+ePfX2mavZzjBM66NUdlQ89iDfIHgJL7cIxbinsLuwbm/zkrjKZ6WEb1NTE8LCwizWcLFUUldBSqnqmBtuuAGrVq0y2X7o0CGb5zIM0zboywnoYuzKIiV3CMVwjL0Z58+fx+7duwEAq1atwqhRo0z2d+zYEUlJSfpyuVJKHD58GAAwcuRIfPEFtcdauXKl2fEnTJiADz/8UF+Dvbi4GIBpad0RI0Zg586dSE9PBwBUVVXh9OnT6N27NzIyMnD27Fm9fQzDuAbjyo4Kof6hKK9zvcfOwt6MPn364NNPP0VycjKKi4vxxBNPtDhm5cqVWLZsGQYOHIh+/frh22+/BQC8/fbbeP/99zFs2DCUlZl/aj/66KOIj49HcnIyBg4ciM8//xwAMGPGDNx0000YO3YsoqKisGLFCtxzzz1ITk7GiBEjcPLkSQQEBGDJkiWYNGkSRo0ahYSEhNa7EQzDWMW4AJhCaECoW3jsXLbXiMzMTEyePBlHjx51qR3ugjv8ThjGWWobarHm+Brc3f9uTYtzrfx9Je5fdz9OPXUKvSJ6AQBGfzIaPl4+2DatdRpuqC3byx47wzDtmjd3v4n7192PPdnadj1rPnkKwG1K97KwG5GYmMjeOsO0I8pry/HGrjcAAMXVxZqOrYRijIWdJ08ZhmFambf3vK33rLX2pEuqSxDoGwg/bz/9NvbYGYZhWpHSmlK8uftNpHZLBaD9wiHjcgIKHf07usUCJRZ2hmHaJWuPr0VZbRn+ecM/AWi/1L+0ttQkDANQVkxdYx1qG2o1vZa9sLAzDNMuuVhxEQAwLHYY/Lz9NPekS6pLTFIdAUMhMFd77SzsOkpLS/V1YRxBbXVGNaV2Dx06hA0bNjhsC8MwQFF1EYJ8g+Dv449Qf+3zy41L9iq4S7MNFnYdbSXsamBhZxjnKaouQkRgBADdwiGtPXajyo4K7tJsg4VdxwsvvICzZ88iJSUFs2bNAgC8/vrrGDZsGJKTkzFv3jwAQGVlJSZNmoSBAweif//+WL16tdmyu8ZYKrW7b98+XHvttRg0aBCuvfZanDp1CnV1dZg7dy5Wr16NlJQUrF692uxxDMNYp7i6GBEdSNhbY1LT0uQp4PpQjFsWAXv2h2dx6KK2ZXtTuqTgrYmWi4v94x//wNGjR/XFvTZv3owzZ85g3759kFJiypQp2L59OwoLCxEbG4v169cDAMrKyhAaGmpSdteYmpoai6V2e/fuje3bt8PHxwdbtmzBiy++iLVr12LBggXYv38/3nvvPQBAeXm52eMYhrFMUVUROnWg6qlapyE2NjWirKashbC7S7MNtxR2d2Dz5s3YvHkzBg0aBIDK9Z45cwajR4/Gc889h9mzZ2Py5MkYPXq01XGsldotKyvDtGnTcObMGQghUF9fb3YMtccxDGOgqLoIcaFxAChEcrb4rGZjX6q6BAmJzkGdTba7S4zdLYXdmmfdVkgpMWfOHDz22GMt9h04cAAbNmzAnDlzMGHCBMydO9fqWJZK7b788ssYO3Ys1q1bh8zMTIwZM8ap4xiGMVBUVaQPxYT6axtjz6/MB4AWwq7vouTpWTFCiDghxDYhxAkhxDEhxDNaGNbWGJfNBYAbb7wRy5cv17eky8nJQUFBAXJzcxEYGIj7778fzz33HH777Tez5ytYK7VbVlaGrl27AgBWrFhh0RZLxzEMY54m2YSSmhJTYdfQiy6oLAAARAdHm2xXPHZXh2K0mDxtAPBXKWUfACMA/FkI0VeDcduUiIgIjBw5Ev3798esWbMwYcIE3HvvvUhNTcWAAQMwdepUXL58GUeOHME111yDlJQULFy4EP/3f/8HwLTsrjHWSu0+//zzmDNnDkaOHGnSJHvs2LE4fvy4fvLU0nEMw5inrKYMTbLJJCumvLYcTbJJk/HzK8hjjw4yFXY/bz8E+AS4PBQDKaWmPwC+BXCDtWOGDBkim3P8+PEW2xjXwr8TxlM5U3RGYj7kp4c+lVJK+cbONyTmQ5bVlGky/pu73pSYD1lcVdxiX/Tr0XLGdzM0uU5zAOyXKnRY03RHIUQigEEA9prZN0MIsV8Isb+wsFDLyzIMw5hQVFUEAIZQjBL71siTLqgsgJ+3X4s8dsA96sVoJuxCiGAAawE8K6VsEWCSUi6RUg6VUg6NiorS6rIMwzAtKKrWCXugIY8d0C72nV+Zj85Bnc0mRihhH1eiibALIXxBor5SSvm1reMtIV3QzYkxD/8uGE+mhceucQ2X/Ir8FhkxCu3CYxf0yFoG4ISUcpGj4wQEBKCoqIgFxQ2QUqKoqAgBAQGuNoVhHEJpqqFfoNQKoZjmE6cKrVGXxl60yGMfCeABAEeEEMpy0RellHYVO+nWrRuys7PB8Xf3ICAgAN26dXO1GUw7p7ER8NauDameouoiCAh9DFxzj70yHwOiB5jdFxoQqu+u5CqcFnYp5Q4A5lfg2IGvry+SkpKcHYZhGA9iyhQgPh7417+0HbeoqgjhHcL1zau19NillCioLEDnQPOhmJjgGORX5qOxqVHT5tn2wEXAGIZxCRkZwMaNQLT5iIZTFFUbVp0C2nrsZbVlqGusa7E4SSEhNAENTQ3Iq8hz+lqOwsLOMIxLWLYMEAJ45BHtxzYu2QsAgb6B8BbemnjslhYnKcSHxgMAzpedd/pajsLCzjBMm1NfDyxfDtx0E4VitKa4ulg/cQpQvSat0hCVcgKWsmIUYc8qzXL6Wo7Cws4wTJuzfj2QlwfMmNE64xsXAFPQKg1RKQBmKRTDHjvDMFckS5YAsbHAzTe3zvjNY+yAdhUebYViQvxDEB4Qjqwy9tgZhrlCkBLYuhW44w7ApxUKh9c11qGirsIkxg7o2uNpEGMvqCyAgGgxvjHxofHssTMMc+VQXAzU1gKtld2srDo1jrEDGnrslfmIDIyEj5flp1JCWAILO8MwVw55uizAmJjWGV9ZddoiFKORx55fmW8xvq4Q3zGeQzEMw1w55ObSa2xs64zfvACYglYee0FlgcWMGIWEsASU15a7rLQACzvDMG1Ka3vszQuAKSgNrZ2tR5VfkW9x4lRBn/LoIq+dhZ1hmDZF8dhbTdgteewBoWhoakB1Q7VT4+dXqhd2V8XZWdgZhmlT8vKA0FAgMNBo4/r1gEb9fC1Nnio12Z0Jj1TVV6GirsJ2KCaUWmCysDMMc0WQm2smvv7ZZ8BLL1EupJNcqroEf29/BPkGmWzXol7Mb3nUvL5bR+uVT6ODo+Hr5euy1acs7AzDtCAvD9i+vfXGbhGGmTCBFP/YMafHz63IRWxIbIvuRlpUeFzw8wJEBUbhtj63WT3OS3ghLjQO58vZY2cYxsWcOEEa260bcN11wLlz2l/DrMd+ww30unmz0+Nnl2eb9aid9dh3nN+BH8/9iNkjZyPYL9jm8QmhCeyxMwzjelasALZtA6ZOpc/Z2dqOL6UFjz0uDujTB9i0yelr5JTnoGvHri22O+uxz0ubh+igaDwx7AlVx7ty9SkLO8Mwes6doxWhL79Mn/PztR2/pIRWnZrNYZ8wgeI/1Y5nrUgpyWMPaemxRwVGAQAKq+zv0nbo4iFszdiKF0a9gEDfQNsngIQ993Iu6hvr7b6es7CwMwyjJyMD6N4d6KxL+tBa2K3msE+YANTUADt2ODx+cXUxahtrzXrsUUFR8BbeyL2ca/e4R/KPAABu7qm+all0UDQkpH4lbFvCws4wjJ6MDPLYIyIALy+goEDb8a2uOr3uOsDPz6k4e3Y5xY7Mxdi9hBe6BHdB3mX7OxudKzkHAaFPY1RDVBB9Q7hUdcnu6zkLCzvDMACAsjIq0JWURA2mIyPb2GMPCgJGjXIqzp5zOQcA0DWkpccOADEhMcitsN9jP1d6Dt06doO/j7/qcyIDIwE4FvpxFhZ2hmEAkLcOUCgGoF6kWgu7zVWnw4ZRao6D+ezWPHaAGk076rF3D+9u1zmKsLPHzjCMy1CEXSmnGx2tfSgmLw/o2JGcc7NERQENDUC5Yy3scspzICDQJbiL2f2xIbEOxdhZ2BmG8UiaC3vnzq3jsVut6hhJYoiiIofGz7mcgy7BXeDr7Wt2f0xwDAqrCu3KVKmur0bu5Vy7hV0pQsbCzjCMy8jIIG86PJw+t0YoxmwOuzGKsF9yTAyzy7PNZsQoxIbQU+VixUXVY2aWZgIAksLs6wzi7+OPEL8QFnaGYVzHuXMUX1dW4kdHA5WV9KMVqj12B4U953KO1TouMSH0VMmrUB9nP1dCy2/t9dgBCsewsDMM4zKUVEcFJZddqzi7xVWnxmjhsVvIiAEMHrs9cXYWdoZhPBIpgcxMU2GP1pUc1yocU1ZG64+sCnuEroa6A8JeWVeJ0ppS6x57sM5jtyMz5lzJOQT6Btos1WuOqKAovbA3NjWitqHW7jEcgYWdYRjk59NK/tYU9kJdOndna/oYGkpJ9A4Iu60cdgDoHNQZXsKrpcf+6KPArFlmzzlXShkxzatFqsHYYz+cfxgdFnbA+tPr7R7HXljYGYbRV3HsbhRt0DoUoyS6KNEWswhBBzgi7OUk7NY8dm8vb0QHRZvG2L/7Dli2DFi6FGhsbHGOI6mOCpEdDMJ+ruQcJKTVyV2tYGFnGKZFqiOgfb0YRautCrtygAPCrixOsiWcJrnsVVXAX/4C+PsDpaXAoUMmx0opSdjDHBT2wEhU1leiur5aH6u3N7vGEVjYGYbRC3tiomGbvz8QFqa9sEdEWD/OYY9dRSgG0JUVUIR94UIgKwv497/p89atJscWVBagqr7KcY9dt0ipqLoI50rOIaJDhL58cGvCws4wDLKyKKbeoYPp9s6dtQvFtLbHnlOeg7CAMAT5WVrWSsQGx1IoprgYWLQIuO8+4M47gb59gZ9+MjnWmYwYwHT1qTMhHXthYWcYBvn5QBczq/C1XKR06RIVbwy21XzIQWE/XXxaVfXFmJAYFFQWoP6TZZSmo0yajhuH0n3bceHSWf2xZ0vovbPCXlhZyMLOMEzbUlBgPltFa2GPjDQsgLJIZCTNtDY1qR67sakRuy/sRmq3VJvHKrns+f/+ABg5Ehg4kHaMG4e7J1XjumWj9ccezDsIf29/9OjUQ7UtxijCfrHiIrLKsljYGYYxICXwxBNAWlrrjF9YSPW3mqNlKKaoSEUYBqCDmppoMlMlRwuO4nLdZYyMH2nzWCWXPbcoE3jySf32g307YVMPIKMmT59hcyDvAAZ2GWix9owtlJrshy4eQkNTAws7wzAGcnOBDz8E7r7bkA+uJZaEPTqaQtH1GnR3Uzx2mziw+nTHeeq6NCp+lM1jFY89r2tH4H/+R7/9n0c/hNBVC96bsxdNsgm/5f2GITFDVNvRnPCAcAgI7MvdB8DxkI69aCLsQojlQogCIcRRLcZjGMaUM2foNT8fmDHD4XLlZqmpAS5fthyKAbTx2i9dUpERA6iu8NjY1AipuxE7LuxAbEisuhh7nR8AIHf8CEr9AU2SfnnsSzxdPQC+jcDe7D1IL07H5brLTgm7t5c3OnXohAO5BwB4mLADWAFgokZjMQzTjPR0en36aeCbb4CVK7UbW/kGYCkUA2gn7JY89l9zfsWPZ3+kDyo99qEfD8X/bvpfAMDO8zsxKn6UqtWhnX/aA+8mILufYSHT4t2L4ePlg9lRtyHlIrA3a5dejIfEOi7sAMXZqxuq4ePlY3XxlJZoIuxSyu0A2r5jK8NcIZw5Qxklb74JxMcD33+v3djWhF3x2C+qr3JrlsZGCulYEvaFvyzEzM0z6YMKYW+STTiSfwTv7nsXG89sxIXyCxgZZzu+DgA+3/4X3Sq9kRVgqNuyLXMbJlw1AbE9BmF4NrD/4m/Yl7MP/t7+6BfVT9W4llAmUBNCE+Dj5ePUWGrhGDvDeABnztByf19f4KqrgAsXtBtb8cbNCbtSsMtZYS8tpflQS8KeGJaIzNJMCq2oEPbi6mI0ykY0ySbc9/V9ANTF11FRAWzejMSALsgqywJAq0szSjPQI7wH0L07hucAlY3V+OLYF0iOTnZ44lRBEfa2CsMAbSjsQogZQoj9Qoj9ha0x+8Mw7Zj0dKBnT3rfrRuQna3d2NaKcynCnmt/NzkTbNWJSQxLREVdBYqri6lvnr+/VWEvqKSn0fCuw1FSU4Ig3yAkRyfbNmTTJqC2Fond+usbaBRWFaKqvgpJ4UlAUhKG6+7txYqLTsXXFdq1sEspl0gph0oph0aZcw0YhjFLUxMJew9dKnVcHJCTY7ZelUNYC8UEBFBZgTz7+z+bYGvVqTLpmVmaqaoQWH4FJdfPHzMfPTr1wJjEMerCHN98A0REIKHnMOSU56Cusc60hktICHr4RKFTE02qOhtfB9q5sDMM4xi5uVRSV/HY4+JI1LUsp+vrSxVzzRET0/rCnhiWCAD68IgtYVc89riOcdjzpz34z+3/sW1EfT1NTtxyCxI7dYeERHZ5NjJKqFCOIryi+1W4ppSWx2rhsUcFRpmM3xZole64CsBuAFcLIbKFEH/SYlyGYQwZMYrH3k2XWKFVnL2ggLx1SwklWgq7pXRHRdiV8IhaYY8OjkZEYATCAsJsG/H66xTsv/tuk+tllGaY2IDu3THhrERUYBT6dXZu4hSAvkGHxwm7lPIeKWWMlNJXStlNSrlMi3EZhjHksBt77IB2cXZLi5MU2sJjDwsIQ0f/jqqFPb8yH17CC506dFJnwO+/A/PnU7GvG29EQpgh9JNRkoHOQZ0NxcO6d8czP5Qi48nT8PP2Uze+FW7rcxuWTF6CQV0GOT2WWjgUwzBuTno6pToqgq61x65W2J1ZFHXpEsXrAwPN7xdC6DNjAKjy2KMCo+AlVEhYXR0wbRoQHg68/z4AasbhJbz0HrtJjfTu3eHV2ISgfG0yuIP9gjF9yHSHOjA5Cgs7w7g5Sqqjtzd97tSJyutq5bEroRhLxMQAtbV2lW5pgVInxpq2JYQmGGLsERFASQnQ0GDe5soCRAdHq7t4Who10HjrLf1XBj9vP3QN6YqssqyWVReVNlJKWykPhIWdYdycM2cMYRiAxLFbN209dmt9SJWUR2fCMWrqxLTIZZeSxN0MBZUF6ptLK4Zfc02L66UXp+N82fkWHjsAFnaGYVqHpibg7FnDxKlCXJw2wl5bS3VibHnsQNsIe3ltOUprSulrCWBR2PMr89ULu4VE/YSwBBzIPYBG2Ug57AqxsRT7YmFnGKY1UFIdzQm7FqEYaznsCloJu60CYCaZMYqwF5uPcxdUFiA6SGUopqCAFjw16/CRGJqI2kYqK2DisXt7U49AFnaGYVoDxSs37kUKUCgmN9f5RUpKOQF3CMUoi5SyyrKsCntVfRUq6irs89g7d24R4NenNwKmHjtA4RgWdoZhWgNFTBVxVVAWKTlbw0WNxx4SQtksjgp7YyNFVNSEYgDbHruSw65a2C3MDispj97CG3Ed40x3du9OMTAPhYWdYTRAy/roxig1WmJjTbdrlfKoRtiFcC6XvaSE7o8tYe/UoROC/YJVC7vqUIyF2WHlQRIXGtey0FdiIqUBlZeru4abwcLOME6yeDEwYIB2tVuMycujkG9z4dVqkZK1yo7G2CPsNTXAqVOGz8rDw5awm+Syh+lWkraixx7XMQ4CwjS+rt+pu8FaltFsQ1jYGcZJdu8Gjh2jV63JzQW6dAG8mv2laqU7hYWAj49BRy0RE6O+wuNHHwH9+wMZtFIfP/9Mr3362D5Xn8vu7U1G6YS9sakR//r1X6ioq9AXALM7xt4Mfx9/9I7sjYHRA1ueEx9Pr+fPq7uGm9E2Vd8Zph2jiOuaNcAoFSXB7SEvr2UxNFE8AAAgAElEQVR8HSDNCwx03mNXVp3aWhQZGwts3KhuzOPHaV3RsmXAq68CK1aQ0CerqKqbGJaInRd20odOnfTCvi1zG57c8CSq6qtQ11gHQKWwV1YCVVUWv5LseGQHOvh0aLmDPXaGubJR/vbXrqW8cy3JzW0ZXwdIiO3NZZeSxPXNNw3bLl60HYYB6OFSUUE/tsjMpNfly+mbzN69wEMP2X54AEB8aDxKa0pxufayibBvz9oOAPj21LcoqCxAiF8IOviaEeTmWCs2D4rrmx0nJoa+JrGwM8yVR0MDedVXXUXe87592o5vyWMH7F99WlQEHDkCzJ4N/PorsHUreeHDh9s+156Ux4wMiqfn5QEPPkhRlfvuU2ejkp1yofyCWWHfeWEnjhUesy++Dqh7ehnj4wN07eqxoRgWdoZxgtxc8tIff5xqmq9Zo93YdXWU/23OYwfs99iVGHlTE3DvvVTosHdvUw/eEmqFvakJyMoiQY+NBX77DbjxRponUENcqE7Yy3TCXlKC2oZa7Mneg+sSrkOTbMKWc1vU14mx4bFbN0aj5b0ugIWdYZxAcej69wcmTCBh1yr1UclRt+Sxx8WR0NbXqxtPEfZXXqGKkQ0N1FAoJMT2uWqFPS+PHkg9egB/0nVlmDZNnX2AeY/919xfUdtYi2eGP4O4jnGQkK3vsQMs7AxzpaL83cfHA5MmkbealaXN2JZy2BXi48lDVputohx3333AV18BmzebFhezRteu9GpL55T4elISMHMmfRu47TZ11wCA2JBYCAiDx15cjO2ZlFYzOmE0br36VgBA50A768Q4Iuzx8fQPbq1FCq0ICzvDOIEidHFxQD9ds52TJ7UZ29KqUwV7EzcUYY+JAaZObVHs0Crh4ZSJo6QwWkLZn5hIx8+cSSEqtfh6+yImJMbgsTc1Yfu5begX1Q+RgZG4tTcJu+pQTEEB1TgOClJvhEJcHFVJUx4OHgQLO8M4wYUL1Cs0JMSQp33ihDZj2/LYHRH2iAiqh+UI5sqnFBUBDzxAE7KAwWNPSHDsGgCFYxRhb/ACdubswR8S/gAAuC7hOtze53ZMuGqCusHU5nOaNcRzUx5Z2BnGCS5cMPz9R0WRcGol7Hl5lHFnKYrgiLBbekioobmw//ILpU/+5z/Ae+9RrD8jgyZKO6jIRLREfGg8hWLCw3GoC1DRUKkXdl9vX6y9cy1GxatcMFBQ4NjEKWBYpMTCzjBXFsbCDpDXrqXH3qWLoXNSc0JC6NtCWwp7ZqahdMJDD1G7u7lzaQ3Qr7/S/uaVKO0lrmMczpedhwwPxx5dTZxr4651bDBbff+sGqL7xXpgyiMLO8M4gTlh1zLGbkuI7Unc0ELY6+ponLIy8t6nTweefpr2b9tGHnuSmdIr9hAXGofqhmoUB3vj11gg2jvUtPrijh3mZ4w/+YSC+sY447FHRtKTiz12hrlyqK4mh1D5xg5QXvilS1b7MKsmN9fyxKlCXJw6h1Ip8WtrPGsYd4w7coTeJyeT/iUnA1u2kAZq4bEDwAXfKvzaFRjmHWdoBF1UBFx3HdCrF/D3v9PkpsLy5VSRbdMm+iylcx67I8t73QQWdoZxEKVOS3OPHdAmHKOlx15YSOLurMcOkLAfPkzvlfovY8dSsa+GBm08dgA4XpeDk5HAsAYjj/vkScrx7N4dePFFYP58wz7lps+cSYZUVFCpSUc9dkD9k9PNYGFnGAcxTnVU0ErY6+pIjG152PHx9O2gutr6cbYybNQQH0+TuefOAb//TimQSn772LGGdG+tPPZvz22AFMDQio6GnUo94HXrgGuvpRlcgG5WUREwZgxVIVuyxLkcdgUll93D4OqODOMg5oQ9Pp4yQpwV9nyqTKvKYwfo24O1xUZaCLuvL/37zp2jn+RkQxbhH/5A76V0Xtijg6Ph6+WL9afXAwCGXQow7Dx9mgxJSABSUoDPPiMP/vhx2v/CC2TESy8ZjHPWY8/NpW8APp4jl+yxM4yDKMKudDMCyKO9+mr7J1CbmkwbdShhHjUxdmNbLKGFsAOGjnFHjpiW4Q0PJ50VwnTOwRG8hBe6deyGyvpKJFT6IqrI6OvIqVNUr8DHhy54+TKl4ihP0j59aBK1Y0fgySdpmzMee2Ii/XIOHHB8DBfAws4wDnLhAmlGQIDpdkdSHt9+m8Tx3/+mcadPJ8d0wADr51kT9qYmquAopUHY1RbjskT37qRxlZUt66vffz8wbpzjC6CMUeLsw6rCTLsonTpFT04AGKhrkHH4MN3w4GC6IUlJwPbthmB/tMpVqua4/Xby+J99VvuazK0ICzvDOMj586ZhGIU+faheTGWl+rF27SLnc9o0Ov/8eeCHH2yv4FS+LZib31u0CLj+euDLL0nYO3e2b3m/Obp3p6gE0FLYZ86kzBgtUOLswxqiDcLe0EDVyxRh79+fviIpwt67tyH8kpBAaZHLl5v/JaklLAx4/XVgzx7qGOIhsLAzjINkZZnPAFEmUI37ftoiPZ2qQy5cSJl827eT92uLgAD61tDcYz9/Hpg3j94vWaIuw0YNSmaMEIbaOK2BXth9EgzCnplJy1sVYQ8MpImFw4cpxt68915sLPDww46VEzDmgQeoNdbs2WZ7sLojLOwM4wBSks6Y86h79aLXM2fUj6U4oi++SDXMU1LU22Iu5VFZNPTYYxSO2bdPW2Hv0cOxulpqGdFtBGJDYjE0uCeJqZSGJ6VygwEKx+zaBeTkqGuq6ghCAO+/T6GYQ4da5xoaw8LOMA5QWEgphuYyQHr0oNfTp9WPVVFhOM9emqda//QT8N13lOI9fz7NMxYUaCvsavqXOsOtvW9FzswchHSKoUVI1dUGYVc8doCEXam53lrCDtA/+MIFdV+j3AAWdqZd8+OPVMdEa5QqhuaEPTCQYt9qPfb0dHp1VNh79KBMFSWr5pdfyMl86imaLJ0yhbZrIeydOlGq+K23Oj+W6gsC5LWfPk2fIyMN+5UJVADo27d1bQkMbN3xNYSFnWm3ZGZS3FoJS2g9NmA5Z7tXL/UeuyLsV13lmC19+pBTq9h04gTF/pUKizNm0KtxWqajCEE1YR54wPmxVBERQa9Hj5pmxCgoMSs/P8PXCYaFnWm/KCmHy5cbFihqhdIlyVLWSs+e6j32s2cpucPRhT3NV7ueOGEalZgwgdbx3HWXY+O7lHHj6In30EO03LW5sMfGkvj37OlRC4haGxZ2pt2iCGt0NDWbrqvTbuzMTMo779jR/P5evSh6UFRke6z0dFrU42j+tyLiJ09SOOb0aVNhF4JyzC3Z6taEhgLffku5o8XFphOnAP3jHn4YuOce19jnprCwM+2W06dJF5YupWy4jz/WbmxbdcftyYxJT3c8DAPQAyY6mjz1jAwKy7TmPGKb068fsHIlFaYfNqzl/tdfpxICjB4Wdqbdcvo0fUOfPJk84l27tBvbUqqjglK3RU2c/exZxydOFXr3JmFXShm0K2EHaAa4tBQYP97VlngEmgi7EGKiEOKUECJdCPGCFmMyjLOcOWPwnPv1M9SJchYpKcZuzWNPSiIH05awl5RQuMZZYVfKGChx9t69nRvPLQkOdrUFHoPTwi6E8AbwPoCbAPQFcI8QopXzjhjGOjU1JL6KsPfta4hBO0tREYV8rQm7nx/ttxSK+eEHYOdO8tYB50IxAAl7aSmQlkZhmfBw58ZjPBstPPZrAKRLKc9JKesAfAGgrbJcGcYs586RZ62ERPr2JbFXUgKdwVaqo4KllMfVq4Gbb6ZslXXraJsWHjtAtVraXRiGsRsthL0rAOMFzdm6bQzjMhRBNfbYAW3CMYqw2yrQ1asXeexKAwqAPPUHHgBSU2li9//9P9rubAq2IuZ1dSzsjDbCbq7CjmxxkBAzhBD7hRD7C5XOJgzTSijCrnjsitgdO+b82EoOuy2PvWdPCtnk5Rm2Pfww2bJhA7B2LVVbjIlxvu5K166GEDQLO6OFsGcDMK6L2Q1AixbiUsolUsqhUsqhUc4UvmcYFZw5Q2VqQ0Ppc2goiZ9WHntoKFV0tYbybUF5yBQXU0PpBx+k81NTqaTuwoXO2ySEYcK0XU6cMnahhbD/CqCnECJJCOEH4G4A32kwLnMF8N57wOjRpuEKLVBSHY3p29d5YZeSJjzVrBJtLuznztGr8UTpH/9IXrwWKJ46e+yM08IupWwA8BSATQBOAPhSSqnBF16mvXPwIDVn2LHDNFyhBadPt1yk2LcvpQPa2winrIzKEkyYQNkmGzeqm+yMi6N66YqwKxkwrVXSZMoUaqzRlWe4rng0Ka4gpdwAYIMWYzFXBtXVwH33GT6fOKFN9UGAOhFdvGhe2KuqqMSt2rosFRV0Xm4uifm991IF11tusX2ulxd9a1Cqzba2sE+dSj8MwytPGZewcCGJ+ZIl9NneHqHWUHLHzYViAPvCMUeOkKh/9BF53h98QHVn1HrFximPZ89Sjjmvs2FaGxZ2xiWsWQNMnEg9PkNCDEvhteDIEXpt3rpNiT3bI+zKsddf71iHtauvpth6fT0Ju7MLkRhGDSzsTJtz4QKFJyZMILFUlsNrxaFDVIu8ucceEUEesz0pj8ePU5zc0ZK6V19NPZgzMkjgWdiZtoCFnWlzfvqJXpV6TloL++HDwIABVKulOQMGUFlvtRw/TumD5sZSgxLn//13IDubhZ1pG1jYmTZnyxbKMe/fnz737k1ZMWVlzo8tJXnslppBp6SQx15fr26848ed67imCPumTWQbN/lh2gIWdqZNkZKEffx4Q8y6eQcgZ8jOpoqJxq0wjRk4kOqVK5kq1rh8mTJonBF2pUXnxo30mT12pi1gYWfalGPHgPx807Laxh2AnOXQIXq15rEbH2cNxR5neyRffTWQk0PvWdiZtoCFnWlTtmyh1+uvN2zr3p3K3GrhsR8+TK8DBpjff/XV1IJOOc4aSkaMFsIOUD2Yzp2dG4th1MDCzphFSuDzzwGt67Vt2kRx5/h4wzYfH8pg0ULYDx2ihUQhIeb3+/pSGqQlj728nFroNTSQsPv6Ou9lK3H2q65yLGWSYeyFhZ0xS1oarQz98EPtxszNBX78keqjNEerzJjDhy3H1xVSUug4c/Vp3ngDmDEDeOcdEvarr6YHjzMoHjuHYZi2goWdMctrr9Hr0aOOnZ+ZSR19jPnkE+pgNH16y+N796Y875oax64H0GRnerrl+LrCwIH0TaR5fZraWnqQCQHMnQvs3et8GAYw9dgZpi1gYWdacOgQhUy8vR0T9owMipuHh9PCni+/pMJbH39MsXVzBbQGDKBj7Fk8tHIlPSSUol5Kfroajx1oGY5ZvZoEf8kSGrOwUBth79kTGDuWVtoyTFvAws604J//pBj19OlU56Suzr7zjx6lMMeTT1Kq3733An/9KzWomDHD/DnXXEOve/eqv87q1cDSpcDbb9M3gVdeoUnYYcOsn6cIv/EEqpQUfunTB/jTn4D582m7kmvvDL6+wNatphPGDNOasLAzJmRkkIf92GPAqFE0iWipIbMl0tPp9ZVXgG3bgEGDgLfeAqKizMfXAWoz17mzfcJ+/jy9zplDNWc2bQLefx/o0sX6eaGh9E3C2GPfvRs4cAD4y18oFDNzJt2HKVPU28Mw7gILO2PCm29SudlnnzUU0bK3nVx6OnUX6tSJPP+NG+kh8eKL5FGbQwhg+HD7hD0rC7jzThLqlSvpYfToo+rOHTSIhFzh668pDfL+++mzjw9wxx3kbTOMp8HCzugpLKSGEg88QGVpe/cmkbc3zp6ebhpHj4wEfvmFHhbWGD6cVoSWlNi+Rnk5Tc4OHUqVIv/yFwrJqGX4cKq2WFREn/fuJbHnkrpMe4CFndHz7ruUlTJrFn0OCKBMDns99rNn1XUYas7w4fS6f7/tYy9coNf4eGqt9/bb5HGrRYnp79tHdWMOHDBcn2E8HRZ2BgB1CnrvPeDWW02bIffvb5+w19dTqqMjwj5sGIVk1IRjsrLo1Xihkz0MHWq41tGj1NGJhZ1pL7CwMwCADRsoBDJzpun2fv1o8lRtfnlWFmWoOJKzHRpKDxU1wq5MnCYk2H8dgGL//frRtZTrsbAz7QUWdgYAeeVeXi1TBfv1o5xuNdUQAUNfT0c8dsAwgWpuVagxWVk0sWkrA8bWtfbtA/bsoXmApCTHx2IYd4KFnQFAy/mTkiiuboy9mTFKqqMzwl5YSOEca5w/D3TrRg8jRxk+HCguBr79lt5zHRemvcDCzgAgYVfK5xqj1EpR23UoPZ2qGEZHO2bH0KH0evCg9ePOn3c8DKOghF5KSzkMw7QvWNgZNDbSClNzwu7nRxkkmzerGys93bkqhv36kRduq6xuVpbjE6fG1woKovcs7Ex7goWdQUYGlQ0wzoYx5o9/JA9ayUSxhqOpjgodOlDRLONvCI2NwGefkX0LFtBq2Jwc5z12b2/DNwRbZQgYxpNgYfcgGhuBggL6qa3VblylXK45jx2gFEgA+O472/adPet8FcPkZIOwNzVRAa0HH6QH0HvvURimqcl5jx0AHnqIxg4Pd34shnEXWNg9iHvuodh1dDSJn1LV0FlsCXuvXrTv22+tj5OTQ56/Mx47QP+2c+eoDO/hw7RqdcECQ+OPTz6h47QS9k8/dX4chnEnWNg9hLo6YP164MYbgaefpph4Wpo2Y584QWmDYWGWj7n1VuDnn02X+zc0AF99RfvCww0lbrUQdoAWDimt9B59FJg0CejYkbx2wPlQDMO0V1jYPYQ9e4CqKuCJJ6gJRmgosGKF/eO89x7wt7+Zbjt50nJ8XeHWW0nIN2wwbHvlFSrCtX8/vU6fTg0qRo2y3y5jFGH//XcS9n79gJgYSsW8/XZDA4+4OOeuwzDtFRZ2D+HHHylbZMwYmmC8805g7VoKV6ilrIxK3C5YYMg3l9JyqqMx11xDXr1xOOa776hOy/nzwEcfAYsX00PDUgVHtcTHk2e+bx+FYcaPN+y75x56jYwEAgOduw7DtFdY2D2ELVtIXEND6fO0aeTBr12rfowVK6gmjBDU2xMALl4kwbcl7F5ewOTJVPO8vp4W9hw+DNxwA2WXaIkQ5LWvXk01XIyFfdw4qtuuRXydYdorLOweQFkZea/GAnfttRTLVjvx19RE1RuvvRZ45BES+YsXbU+cGjNpEpXL3bmT4u1SUsZKa5CcDFRW0kPjuusM2318gGXLKAzEMIx5nOy/zrQFaWkkzMbCLgTVTZ83j9IfO3e2PsbGjZSKuHAh1R1fupTa1CkNndUI+/jxFGb5/nuazA0MNJS/1Rolzj5iBBXsMmby5Na5JsO0F9hjdyMspS9u2UIiOmKE6fYJE+j155+tjyslsGgREBtLk4+9egFTpwL//S9NRL73HjXWsEVwMHnP69fTw2bkSOfj6ZZQhN34YcYwjDpY2N2EY8dIOPftM92ek0Nx9D/8oWUjiSFD6Jxt26yPvXYtNVN+/nlDq7elSylGfvo08Oc/q7dz0iTKojlypPXCMACtCH3+ecq0YRjGPljY3YStW2mi8KOPDNuKisgrv3wZePXVluf4+lJWijVhLy8HnnkGSEkxFfCOHckrtremy6RJhvetKey+vpTWqeabBMMwprCwa0RjI2WLNDQ4dr7SWPmrryjbpb6eYslnz1Ja4ZAh5s8bO5Y8aCVW3pyXXqJ9H31EE4/O0qMHhXKCgy3bxDCMa2Fh14CDBykG7udH4ZLvv7d/jAMHqFTA5cvAunXUw3PPHlo+b80zVvY1X4UqJU2svvce8NRT2k5yLlwI/P3vhrAOwzDuhVPCLoS4QwhxTAjRJIQYqpVRnsY335CnvmABrYb8299sdwAypqoKOH6cls0nJgJvvkmifMstwN13Wz930CDKbTcOx0hJK1QXLKDUxkWLHPpnWWTqVHpYMAzjnjjrsR8FcDuA7RrY4rFs2UJlX19+GXjxRVpib2tC05jDhykjZuhQqjSoNJl4913bMXBvb5pYNb7e779T6OXZZ2mSVIsQDMMwnoNTwi6lPCGlVNkNs31SXk49OpW0vAcfpJDKa6+pH0OJrw8ZQitK/fxoAY7aIldjx1KJAKXB8w8/0Ovzz3O7N4a5EmmzGLsQYoYQYr8QYn9hYWFbXbbV+flnmjhVhD0ggDzlzZttt3dTOHAAiIqiHp7du9OK0Jkz1dtwyy30+tVX9PrDD5TxEhOjfgyGYdoPNoVdCLFFCHHUzM+t9lxISrlESjlUSjk0KirKcYvdjC1bqChXaqph2xNP0CTqypXqxjhwgLx1xbu2t+lDjx4Uxlm1iiZfd+6k8r4Mw1yZ2Iy+Sil57Z8Vfvqp5eKh0FAS6t27bZ9fXU0Tp7fa9ZhsyT33AH/9K7BkCaVKTpzo3HgMw3gunO7oBHl5tGLU3LL31FTyxOvqrI9x+DCFcpzNCb/rLvL4586l1MuRI50bj2EYz8XZdMfbhBDZAFIBrBdCbNLGLO349FNqf/bQQ441prDGTz/R6/XXt9yXmkp9Sa3F2S9epLCNr2/LOjD20rUrfXOoqqLSts3LDzAMc+XgVCKclHIdgHUa2aI5ly4Bjz9uaMiwahXFnrWaVPzlFwq7DBzYcp8Sc9+9Gxg+vOX+06fJlsJCWlnapYvz9txzD03mcnydYa5s2nUoZulSoKYG2L6dims1NNCKTq3YuZPqm3uZuYuxsdQMwlycfe9eOq+ykvLPtYqH33cfMHs2vTIMc+XSboW9oQF4/30Kk/TrB1x1Fa2Y/Ne/qHGFPTz2GOWn19cbthUXU3zdWiw7NbWlsG/bRqGS0FBg1y5a2KQVwcHAP/5hf1YNwzDti3Yr7N98A2RnA3/5i2Hb7Nm0oMi4gqItLlwAPv4Y+Owz4OGHDTXTFcG21rg5NZXOz842bHvtNSAigkS9Rw/1djAMw6il3Qr7O+9Q3RXjMrODB1MGy9tvUyaKGv7zH6q98vjjlJc+axZt37GDlupb87iN4+wAjbNvH8XAo6Pt/icxDMOowqOF/fJlqqdy8aLp9mPHaGLzySdbNlqePh3IzaX4uC2kpKyaUaOADz4gcV+8mDJddu6kFEVlYtYcKSm0EnXXLvp85gxQUmJ+MpVhGEYrPFrYFy+mUEuPHpS/reSMf/wxpRA+9FDLc26+mcR2zRrb4+/bB5w6ReMIQaVqO3WixhX79tnOFffzo4fCJl0S6N699MrCzjBMa+Kxwt7YSFkvqakUbnnlFWDOHFrJ+e9/U29Pc5ULgoOBm26idnGWeowqfPoplQu44w76HBYGzJ9P3wZqa63H1xUmTQJOnAAyMkjYg4OBvn3t/ucyDMOoxmOFfdMmmpj861+B1asp7LJoEXnwJSXAjBmWz506lcIxe/ZYPqa0lGLqt91GbeQUHnuMOggBlLJoCyXGv349CfvQoS3DQwzDMFriscK+ZAlNQE6ZQp9ff50Ed+lSCs2MGWP53MmTKUxiLRyzeDFl0CiTpQq+vpQh8/e/q5sA7dmTftaupfIBHIZhGKa18Uhhz8mh9nMPP2xozxYYSBks/v7ktZtbNKTQsSNlpqxZY9rpqLCQPhcXk7DffjtNgDbnmmuAF15Qb+/kydS6rr6ehZ1hmNbH44S9sZFqlTc1USs5Y4YNAwoKgKeftj3OnXdSKGfHDvp88CCVGkhNBf78Z8q4mT9fG5uNUy617D3KMAxjDo8Sdimp1+aXX9IKy6uuanmMcTzcGrfdRhOZn35Knz/8kMIzOTnAF1+Q8A8YoI3do0cDISFUqKtrV23GZBiGsYRHCfvcuSTAs2dT2zdnCAqiSdQvvyQv//PPqXH06dOUVfPuu9rYDNADY9YsbgDNMEzbIKRxkLmNGDp0qNy/f7/d523YQC3nFi/WppdnWhr1Cx03Dti6lVaIOls+l2EYprUQQhyQUg61eZwnCbvWNDVRj9GsLAq7HD7MzZ8ZhnFf1Aq7R4VitMbLi6o2ApT3zqLOMEx7wKlGG+2Bp56ifPVp01xtCcMwjDZc8cLeuTPw1luutoJhGEY7ruhQDMMwTHuEhZ1hGKadwcLOMAzTzmBhZxiGaWewsDMMw7QzWNgZhmHaGSzsDMMw7QwWdoZhmHaGS2rFCCEKAWQ5eHokgEsamtOaeIqtnmIn4Dm2eoqdANvaGrSWnQlSSjPdnE1xibA7gxBiv5oiOO6Ap9jqKXYCnmOrp9gJsK2tgavt5FAMwzBMO4OFnWEYpp3hicK+xNUG2IGn2OopdgKeY6un2Amwra2BS+30uBg7wzAMYx1P9NgZhmEYK3iUsAshJgohTgkh0oUQL7jaHgUhRJwQYpsQ4oQQ4pgQ4hnd9k5CiB+FEGd0r+GuthUAhBDeQoiDQojvdZ+ThBB7dXauFkL4udpGABBChAkh1gghTurubaob39P/1f3ujwohVgkhAtzlvgohlgshCoQQR422mb2PgnhH9zf2uxBisIvtfF33+/9dCLFOCBFmtG+Ozs5TQogb28pOS7Ya7XtOCCGFEJG6z21+Tz1G2IUQ3gDeB3ATgL4A7hFC9HWtVXoaAPxVStkHwAgAf9bZ9gKAn6SUPQH8pPvsDjwD4ITR59cALNbZWQLgTy6xqiVvA/hBStkbwECQzW53T4UQXQH8BcBQKWV/AN4A7ob73NcVACY222bpPt4EoKfuZwaAf7WRjYB5O38E0F9KmQzgNIA5AKD7+7obQD/dOR/oNKKtWIGWtkIIEQfgBgDnjTa3/T2VUnrED4BUAJuMPs8BMMfVdlmw9VvdL/cUgBjdthgAp9zAtm6gP+RxAL4HIEALKXzM3WcX2tkRQAZ080BG293xnnYFcAFAJ1BXsu8B3OhO9xVAIoCjtu4jgI8A3GPuOFfY2WzfbQBW6t6b/P0D2AQg1ZX3VLdtDcgJyQQQ6ap76jEeOwx/PArZum1uhRAiEcAgAHsBREsp8wBA99rZdZbpeQvA8wCadJ8jAJRKKRt0n93lvnYHUBfwSyIAAAKuSURBVAjgE13YaKkQIghueE+llDkA3gB5aXkAygAcgHveVwVL99Gd/84eAbBR997t7BRCTAGQI6U83GxXm9vqScIuzGxzq5QeIUQwgLUAnpVSlrvanuYIISYDKJBSHjDebOZQd7ivPgAGA/iXlHIQgEq4QdjFHLr49K0AkgDEAggCff1ujjvcV1u45f8HIcRLoJDnSmWTmcNcZqcQIhDASwDmmtttZlur2upJwp4NIM7oczcAuS6ypQVCCF+QqK+UUn6t25wvhIjR7Y8BUOAq+3SMBDBFCJEJ4AtQOOYtAGFCCKWxubvc12wA2VLKvbrPa0BC7273FADGA8iQUhZKKesBfA3gWrjnfVWwdB/d7u9MCDENwGQA90ldLAPuZ+dVoAf7Yd3fVzcAvwkhusAFtnqSsP8KoKcu08APNHHynYttAkCz3gCWATghpVxktOs7ANN076eBYu8uQ0o5R0rZTUqZCLp/W6WU9wHYBmCq7jCX2wkAUsqLAC4IIa7WbboewHG42T3VcR7ACCFEoO7/gmKr291XIyzdx+8APKjL5BgBoEwJ2bgCIcREALMBTJFSVhnt+g7A3UIIfyFEEmhicp8rbAQAKeURKWVnKWWi7u8rG8Bg3f/jtr+nbTnZoMFkxc2gmfGzAF5ytT1Gdo0CfbX6HcAh3c/NoPj1TwDO6F47udpWI5vHAPhe97476I8iHcBXAPxdbZ/OrhQA+3X39RsA4e56TwH8DcBJAEcBfAbA313uK4BVoNh/PUhw/mTpPoLCBu/r/saOgDJ9XGlnOig+rfxdfWh0/Es6O08BuMnV97TZ/kwYJk/b/J7yylOGYZh2hieFYhiGYRgVsLAzDMO0M1jYGYZh2hks7AzDMO0MFnaGYZh2Bgs7wzBMO4OFnWEYpp3Bws4wDNPO+P9uzNlBMM08yQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    seq_size = 5\n",
    "    predictor = SeriesPredictor(input_dim=1, seq_size=seq_size, hidden_dim=100)\n",
    "    data = data_loader.load_series('international-airline-passengers.csv')\n",
    "    train_data, actual_vals = data_loader.split_data(data)\n",
    "\n",
    "    train_x, train_y = [], []\n",
    "    for i in range(len(train_data) - seq_size - 1):\n",
    "        train_x.append(np.expand_dims(train_data[i:i+seq_size], axis=1).tolist())\n",
    "        train_y.append(train_data[i+1:i+seq_size+1])\n",
    "\n",
    "    test_x, test_y = [], []\n",
    "    for i in range(len(actual_vals) - seq_size - 1):\n",
    "        test_x.append(np.expand_dims(actual_vals[i:i+seq_size], axis=1).tolist())\n",
    "        test_y.append(actual_vals[i+1:i+seq_size+1])\n",
    "\n",
    "    predictor.train(train_x, train_y, test_x, test_y)\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "   \n",
    "        prev_seq = train_x[-1]\n",
    "        predicted_vals = []\n",
    "        for i in range(20):\n",
    "            next_seq = predictor.test(sess, [prev_seq])\n",
    "            predicted_vals.append(next_seq[-1])\n",
    "            prev_seq = np.vstack((prev_seq[1:], next_seq[-1]))\n",
    "        plot_results(train_data, predicted_vals, actual_vals, 'hallucinations.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}