Pelevin-generator-train(without_val).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Пелевин RNN - тренировка модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T18:20:34.854793Z",
     "start_time": "2019-11-05T18:20:34.372865Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.nn import Embedding, Linear, LSTM, Dropout\n",
    "\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Загрузка обучающего датасета (текста)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T18:21:31.570320Z",
     "start_time": "2019-11-05T18:21:31.528673Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('pelevin-all_prepared.txt') as input_file:\n",
    "    texts = input_file.read()[:-1].split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76275\n"
     ]
    }
   ],
   "source": [
    "print(len(texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Колдун Игнат и люди',\n",
       " 'Сказочка',\n",
       " '4 мая 1912 года к колдуну Игнату пришел в гости протоиерей Арсеникум. Пока Игнат хлопотал с самоваром, доставал пряники, гость сморкался у вешалки, долго снимал калоши, крестился и вздыхал. Потом сел на краешек табурета, достал из-под рясы папку красного картона, раскрыл и развязно сказал Игнату:',\n",
       " '– Глянь-ка, чего я понаписал!',\n",
       " '– Интересно, – сказал Игнат, беря первый лист, – вслух читать?',\n",
       " '– Что ты! – испугано зашипел протоиерей. – Про себя!',\n",
       " 'Игнат стал читать:',\n",
       " '«ОТКРОВЕНИЕ СВ. ФЕОКТИСТА»',\n",
       " '– «Люди! – сказал св. Феоктист, потрясая узловатым посохом. – Христос явился мне, истинно так. Он велел пойти к вам и извиниться. Ничего не вышло».',\n",
       " '– Ха-ха-ха! – засмеялся Игнат, а сам подумал: «Неспроста это». Но виду не подал.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAV2UlEQVR4nO3dfbBtdX3f8fenoBjUykWuylO9EKkjzigyN4g1tTQqjzYkM2phnHi1mGsTnGpjx1yTTPBxipkktTSWiBHFVlGaaETBIiFaaxvRi+FRglzxVm4ucq+iWGNiQb/9Y/9O3BzO89kP55zf+zWzZ6/9W2uv9d1n7/NZa//Ww05VIUnqwz+YdgGSpMkx9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoa8NLsjvJC1b43C1JKsmB7fGnkmwbUV3/NMkdo6hznvnfluSUUc1PG4Ohr7EZdYgtcZnvT/K2cc2/qs6oqsuWUEclecoi8/qfVfXUUdQ11+uuqqdX1WdHMX9tHIa+NAUz3xykSTP0NRVJXpTkxiTfTfK/kzxjaNzuJP8uyc1J7k/ykSSPGhr/hiT3JNmb5FUzW9VJtgMvA96Q5PtJPjG0yBPmm9+sug5I8rtJvpXkLuCsWeM/m+RVbfgpSf5Hm+e3knyktX+uTX5Tq+NfJjklyZ4kv57km8D7ZtpmlfAzSb6S5DtJ3jdTZ5JXJPn8rFoWfN3D37SSHJTkne1vtrcNH9TGzdT2+iT72t/2lUt4G7UOGfqauCQnApcCrwYeD7wbuHImhJqXAqcDxwDPAF7Rnns68GvAC4CnAP9s5glVdQnwQeB3quoxVfUvFpvfHH4ZeBHwLGAr8OIFXspbgU8Dm4CjgP/U6nheG//MVsdH2uMnAYcCTwa2zzPPlwGnAT8N/GPgtxZYPm15C73uGb8JnAycADwTOGnWvJ8EPA44EjgPeFeSTYstW+uPoa9p+GXg3VV1fVX9qPWR/5BBKM24qKr2VtV9wCcYhBUMwvt9VXVbVf0AePMSlznf/GZ7KfDOqrq7TfvvF5jnAwwC/Iiq+ruq+vwC0wL8GLigqn5YVX87zzR/MLTstwPnLjLPpXoZ8Jaq2ldV+xn83X5paPwDbfwDVXU18H1gJPsbtLYY+pqGJwOvb107303yXeBo4Iihab45NPwD4DFt+Ajg7qFxw8MLmW9+s82e//9ZYJ5vAAJ8sR0p868WqWF/Vf3dItPMXvYR8024TEfw0Ncye97frqoHhx4v9DfSOubOJE3D3cDbq+rtK3juPQy6UmYcPWv8ai8be8+sef6j+Sasqm8y+NZCkp8F/izJ56pq13xPWcLyZy97bxv+G+DgmRFJnrTMee9lsLK9bY55qyNu6WvcHpHkUUO3A4H3AP86ybMz8OgkZyV57BLmdwXwyiRPS3Iw8Nuzxt8LHLuKeq8A/k2So1qf9o75JkzykiQzK6DvMAjeH62yjvPbsg8FfgOY2R9wE/D0JCe0nbtvmvW8xZZ3OfBbSTYnOYzB3+2/rqA+rXOGvsbtauBvh25vqqqdDLaQ/4BBWO5i/h2rD1FVnwIuAj7TnvcXbdQP2/17geNbt9GfrqDe9wDXMAjZLwMfXWDanwGuT/J94ErgtVX19TbuTcBlrY6XLmP5H2Kwc/iudnsbQFV9FXgL8GfAncDs/QeLve63ATuBm4Fb2msb2/kMWrvij6hoPUvyNOBW4KBZfdKS5uCWvtadJL+Y5JGt++UdwCcMfGlpDH2tR68G9gNfY9CH/ivTLUdaP+zekaSOuKUvSR1Z08fpH3bYYbVly5ZplyFJ68oNN9zwraraPNe4NR36W7ZsYefOndMuQ5LWlSTznklu944kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHWk+9DfsuOqaZcgSRPTfehLUk8MfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHdnQoe/v30rSQ23o0JckPZShL0kdMfQlqSOGviR1xNCXpI4Y+pLUkUVDP8nRST6T5PYktyV5bWs/NMm1Se5s95tae5JclGRXkpuTnDg0r21t+juTbBvfy5IkzWUpW/oPAq+vqqcBJwPnJzke2AFcV1XHAde1xwBnAMe123bgYhisJIALgGcDJwEXzKwoJEmTsWjoV9U9VfXlNvx/gduBI4GzgcvaZJcBv9CGzwY+UANfAA5JcjhwGnBtVd1XVd8BrgVOH+mrkSQtaFl9+km2AM8CrgeeWFX3wGDFADyhTXYkcPfQ0/a0tvnaZy9je5KdSXbu379/OeVJkhax5NBP8hjgT4DXVdX3Fpp0jrZaoP2hDVWXVNXWqtq6efPmpZYnSVqCJYV+kkcwCPwPVtVHW/O9rduGdr+vte8Bjh56+lHA3gXaJUkTspSjdwK8F7i9qn5/aNSVwMwRONuAjw+1v7wdxXMycH/r/rkGODXJprYD99TWJkmakAOXMM1zgV8CbklyY2v7DeBC4Iok5wHfAF7Sxl0NnAnsAn4AvBKgqu5L8lbgS226t1TVfSN5FZKkJVk09Kvq88zdHw/w/DmmL+D8eeZ1KXDpcgqUJI2OZ+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfSBLTuuYsuOq6ZdhiSNnaEvSR0x9CWpI4a+JHWk69C3H19Sb7oOfUnqjaEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRbkPfwzUl9ajb0JekHhn6ktQRQ1+SOmLoS1JHDP0hC+3cdcevpI3A0J/FcJe0kRn6ktSRLkPfrXlJveoy9CWpV4a+JHXE0Jekjhj6c7DPX9JGtWjoJ7k0yb4ktw61vSnJXye5sd3OHBr3xiS7ktyR5LSh9tNb264kO0b/UiRJi1nKlv77gdPnaP8PVXVCu10NkOR44Bzg6e05/znJAUkOAN4FnAEcD5zbppUkTdCBi01QVZ9LsmWJ8zsb+HBV/RD4epJdwElt3K6qugsgyYfbtF9ZdsWSpBVbTZ/+a5Lc3Lp/NrW2I4G7h6bZ09rma5ckTdBKQ/9i4KeBE4B7gN9r7Zlj2lqg/WGSbE+yM8nO/fv3r7A8SdJcVhT6VXVvVf2oqn4MvIefdOHsAY4emvQoYO8C7XPN+5Kq2lpVWzdv3ryS8iRJ81hR6Cc5fOjhLwIzR/ZcCZyT5KAkxwDHAV8EvgQcl+SYJI9ksLP3ypWXLUlaiUV35Ca5HDgFOCzJHuAC4JQkJzDootkNvBqgqm5LcgWDHbQPAudX1Y/afF4DXAMcAFxaVbeN/NVIkha0lKN3zp2j+b0LTP924O1ztF8NXL2s6iRJI+UZuZLUEUNfkjpi6EtSRwx9SeqIoS9JHTH05+HllSVtRIa+JHXE0F/Alh1XucUvaUMx9CWpI4a+JHXE0Jekjhj6ktSRRS+4tpG4U1ZS79zSXwJXFpI2CkNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKG/DF6DR9J6Z+hLUkcMfUnqiKEvSR3pIvTti5ekgS5CX5I0YOhLUke6CX27eCSpo9CXJBn6ktSVRUM/yaVJ9iW5dajt0CTXJrmz3W9q7UlyUZJdSW5OcuLQc7a16e9Msm08L0eStJClbOm/Hzh9VtsO4LqqOg64rj0GOAM4rt22AxfDYCUBXAA8GzgJuGBmRbGRuN9A0lq3aOhX1eeA+2Y1nw1c1oYvA35hqP0DNfAF4JAkhwOnAddW1X1V9R3gWh6+IpEkjdlK+/SfWFX3ALT7J7T2I4G7h6bb09rma5ckTdCod+RmjrZaoP3hM0i2J9mZZOf+/ftHWpwk9W6loX9v67ah3e9r7XuAo4emOwrYu0D7w1TVJVW1taq2bt68eYXlSZLmstLQvxKYOQJnG/DxofaXt6N4Tgbub90/1wCnJtnUduCe2tokSRO0lEM2Lwf+Anhqkj1JzgMuBF6Y5E7ghe0xwNXAXcAu4D3ArwJU1X3AW4EvtdtbWtu6M3OEzpYdV3m0jqR158DFJqiqc+cZ9fw5pi3g/Hnmcylw6bKqW6MMe0nrlWfkSlJHDH1J6oihL0kdMfQlqSOGviR1xNBfJY/kkbSeGPqS1BFDX5I6YuhLUkcMfUnqiKE/AsvdmevOX0nTYuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6I+JhmJLWA0Nfkjpi6I+JW/6S1iJDX5I6cuC0C9ho3MKXtJa5pS9JHTH0x8itfklrzYYPfYNXkn5iw4f+tM2sdLbsuGpZKyBXVpLGwdCXpI4Y+hPgVruktcLQl6SOGPoT5la/pGky9CWpI4b+lLjFL2kaDH1J6oihP2Vu8UuaJEN/nXOlIWk5DH1J6siqQj/J7iS3JLkxyc7WdmiSa5Pc2e43tfYkuSjJriQ3JzlxFC9AkrR0o9jS/+dVdUJVbW2PdwDXVdVxwHXtMcAZwHHtth24eATLXtfsmpE0aeP4EZWzgVPa8GXAZ4Ffb+0fqKoCvpDkkCSHV9U9Y6hhXRkO/90XnuXKQNLYrHZLv4BPJ7khyfbW9sSZIG/3T2jtRwJ3Dz13T2t7iCTbk+xMsnP//v2rLE+SNGy1of/cqjqRQdfN+Umet8C0maOtHtZQdUlVba2qrZs3b15leevPfFv5bv1LGoVVhX5V7W33+4CPAScB9yY5HKDd72uT7wGOHnr6UcDe1Sy/Nwa/pNVacegneXSSx84MA6cCtwJXAtvaZNuAj7fhK4GXt6N4Tgbutz9/YXP98MpKgt+VhaQZq9mR+0TgY0lm5vOhqvrvSb4EXJHkPOAbwEva9FcDZwK7gB8Ar1zFsiVJK7Di0K+qu4BnztH+beD5c7QXcP5Kl9cz+/kljYpn5K5Dy/293eXMV9LGZuhLUkcMfUnqiKG/ASzULWOXjaRhhr4kdcTQ32DGtZNX0sZg6G8QcwW9R/hIms3Q36DmOpN3rQX2WqtH6oGhv4EYopIWY+h3avaW/6hXGK6ApLXJ0O/QcgLZ8JY2FkO/M4v19Rvy0sZm6GtOwyuDuVYMo7jks6TJM/T1MMu5queou4pWuvJYi0cnSWuRoa8VW8phof4IjLS2GPqaOH8fQJqe1fxylgSMZmt+sXlKGg1DX2O30L6ASYf7lh1XsfvCsya6TGktsXtH6944Tizzm4Y2KkNf68ZSg9jAluZn6GvNGeUVQ10BSA9ln77GZpyXdt594VmLzn+hHczDwyvt43f/gNYjt/S1oax2B7HfDLTRGfpal5Z7JvBC1xda7hnIrhi0ntm9I7F48K+mC2g1z5dGzS19aYmW8o1g5pvAQt8G/KagaTL0pRUY1VnHyzmb2a4ljYLdO9ISrPZCcSu5EN1Kjw7yqCItxC19aRXm+82BSS1XWi639KUpW8r5BMNb7vMdlrqcrftRnKeg9cktfWkdWO0P0Kz0/IVRfqPw28naYOhLG9ByLmUxu4tquV1Vo+zaWqvzmvSyx1m3oS9tIEs5fHSpK4TZ3w5mz2/aobqc9pXMa9QW2rE/SYa+pEUtdcUx10phdtt80881/6Wc87CUazDNtdzFzPec5VzzaSXdcuNeIbgjV9LIjeM3DhZqm32Y6lJPpJtrePhifvPNd7ndX8sZHreJh36S04H/CBwA/FFVXTjpGiRNz1K7Zka903mp85v0lvekTbR7J8kBwLuAM4DjgXOTHD/JGiStPR4lNDmT7tM/CdhVVXdV1f8DPgycPeEaJKlbk+7eORK4e+jxHuDZwxMk2Q5sbw+/n+SOVSzvMOBbq3j+uFjX8ljX8ljX8qzJuvKOVdX15PlGTDr0M0dbPeRB1SXAJSNZWLKzqraOYl6jZF3LY13LY13L01tdk+7e2QMcPfT4KGDvhGuQpG5NOvS/BByX5JgkjwTOAa6ccA2S1K2Jdu9U1YNJXgNcw+CQzUur6rYxLnIk3URjYF3LY13LY13L01VdqarFp5IkbQhehkGSOmLoS1JHNmToJzk9yR1JdiXZMeFlX5pkX5Jbh9oOTXJtkjvb/abWniQXtTpvTnLiGOs6Oslnktye5LYkr10LtSV5VJIvJrmp1fXm1n5MkutbXR9pO/5JclB7vKuN3zKOuobqOyDJXyb55FqpK8nuJLckuTHJzta2Fj5jhyT54yR/1T5nz5l2XUme2v5OM7fvJXndtOtqy/q37TN/a5LL2//C+D9fVbWhbgx2EH8NOBZ4JHATcPwEl/884ETg1qG23wF2tOEdwDva8JnApxicv3AycP0Y6zocOLENPxb4KoNLYUy1tjb/x7ThRwDXt+VdAZzT2v8Q+JU2/KvAH7bhc4CPjPn9/DXgQ8An2+Op1wXsBg6b1bYWPmOXAa9qw48EDlkLdQ3VdwDwTQYnLk37c38k8HXgp4Y+V6+YxOdrrH/kadyA5wDXDD1+I/DGCdewhYeG/h3A4W34cOCONvxu4Ny5pptAjR8HXriWagMOBr7M4CztbwEHzn5PGRz59Zw2fGCbLmOq5yjgOuDngE+2IFgLde3m4aE/1fcR+IctxLKW6ppVy6nA/1oLdfGTqxMc2j4vnwROm8TnayN278x1qYcjp1TLjCdW1T0A7f4JrX0qtbavhs9isFU99dpaF8qNwD7gWgbf1L5bVQ/Osey/r6uNvx94/DjqAt4JvAH4cXv8+DVSVwGfTnJDBpctgem/j8cC+4H3te6wP0ry6DVQ17BzgMvb8FTrqqq/Bn4X+AZwD4PPyw1M4PO1EUN/0Us9rCETrzXJY4A/AV5XVd9baNI52sZSW1X9qKpOYLBlfRLwtAWWPZG6krwI2FdVNww3T7uu5rlVdSKDq9Wen+R5C0w7qboOZNCteXFVPQv4GwbdJtOua7CwQd/4zwP/bbFJ52gbx+drE4OLTR4DHAE8msH7Od+yR1bXRgz9tXiph3uTHA7Q7ve19onWmuQRDAL/g1X10bVUG0BVfRf4LIO+1EOSzJw8OLzsv6+rjX8ccN8Yynku8PNJdjO4GuzPMdjyn3ZdVNXedr8P+BiDFeW038c9wJ6qur49/mMGK4Fp1zXjDODLVXVvezztul4AfL2q9lfVA8BHgX/CBD5fGzH01+KlHq4EtrXhbQz602faX96OGDgZuH/mK+eoJQnwXuD2qvr9tVJbks1JDmnDP8Xgn+F24DPAi+epa6beFwN/Xq2jc5Sq6o1VdVRVbWHwGfrzqnrZtOtK8ugkj50ZZtBPfStTfh+r6pvA3Ume2pqeD3xl2nUNOZefdO3MLH+adX0DODnJwe1/c+bvNf7P1zh3nEzrxmAP/FcZ9A3/5oSXfTmDProHGKydz2PQ93YdcGe7P7RNGwY/KvM14BZg6xjr+lkGXwdvBm5stzOnXRvwDOAvW123Ar/d2o8FvgjsYvCV/KDW/qj2eFcbf+wE3tNT+MnRO1Otqy3/pna7bebzPe33sS3rBGBney//FNi0Ruo6GPg28LihtrVQ15uBv2qf+/8CHDSJz5eXYZCkjmzE7h1J0jwMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSR/w8d7QBS21mZ6QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('Length distribution')\n",
    "plt.hist(list(map(len, texts)), bins=256);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800\n"
     ]
    }
   ],
   "source": [
    "MAX_LEN = max(map(len,texts))\n",
    "\n",
    "print(MAX_LEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Загрузка токенов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tokens.txt') as t_file:\n",
    "    tokens_string = t_file.read()\n",
    "tokens = list(tokens_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_to_id = {token: idx for idx, token in enumerate(tokens)}\n",
    "num_tokens = len(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[' ', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '|', '~', '©', '«', '®', '°', '»', 'À', 'É', 'Ñ', 'Ø', 'à', 'á', 'â', 'ä', 'ç', 'è', 'é', 'ë', 'ï', 'ó', 'ø', 'ü', 'ň', '́', 'α', 'γ', 'ε', 'μ', 'ς', 'σ', 'τ', 'Ё', 'Є', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'ё', 'є', 'і', 'ї', 'ѣ', 'ה', 'ו', 'ז', 'ח', 'י', 'פ', 'ץ', 'צ', '‑', '–', '—', '‘', '’', '“', '”', '„', '…', '№', '™', 'ⓔ']\n"
     ]
    }
   ],
   "source": [
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "214\n"
     ]
    }
   ],
   "source": [
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Функция преобразования токинизированного текста в тензор"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_matrix(data, token_to_id, max_len=None, dtype='int32', batch_first = True):\n",
    "\n",
    "    max_len = max_len or max(map(len, data))\n",
    "    data_ix = np.zeros([len(data), max_len], dtype) + token_to_id[' ']\n",
    "\n",
    "    for i in range(len(data)):\n",
    "        line_ix = [token_to_id[c] for c in data[i]]\n",
    "        data_ix[i, :len(line_ix)] = line_ix\n",
    "        \n",
    "    if not batch_first: # convert [batch, time] into [time, batch]\n",
    "        data_ix = np.transpose(data_ix)\n",
    "\n",
    "    return data_ix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Определение модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PelevinRNN(torch.nn.Module):\n",
    "    def __init__(self, num_tokens=num_tokens, emb_size=100, rnn_num_units=512):\n",
    "        super(self.__class__, self).__init__()\n",
    "        self.emb = Embedding(num_tokens, emb_size)\n",
    "        self.rnn = LSTM(emb_size, rnn_num_units, num_layers=3, batch_first=True)\n",
    "        self.hid_to_logits = Linear(rnn_num_units, num_tokens)\n",
    "        self.dropout = Dropout(p=0.6)\n",
    "\n",
    "    def forward(self, x, hidden_state=None):\n",
    "        x = self.dropout(self.emb(x))\n",
    "        if hidden_state is not None:\n",
    "            h_seq, new_hidden = self.rnn(x, hidden_state)\n",
    "        else:\n",
    "            h_seq, new_hidden = self.rnn(x)\n",
    "        next_logits = self.hid_to_logits(h_seq)\n",
    "        next_logp = F.log_softmax(next_logits, dim=-1)\n",
    "        return next_logp, new_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Module.parameters of PelevinRNN(\n",
       "  (emb): Embedding(214, 100)\n",
       "  (rnn): LSTM(100, 512, num_layers=3, batch_first=True)\n",
       "  (hid_to_logits): Linear(in_features=512, out_features=214, bias=True)\n",
       "  (dropout): Dropout(p=0.6, inplace=False)\n",
       ")>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = PelevinRNN().cuda()\n",
    "history = []\n",
    "\n",
    "model.parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15000, 30000], gamma=0.1)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Тренировочный цикл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xV9f348dc7IRBGGEJYAQlgQMPGgAMZWpAp1Dp+WK2jKF+tOFqLxVIRsSiCWlvFWkfdCtRRsSwFUVCGRHYYEjBAwkgIewQyPr8/7sgd546Qm9zcm/fz8fDhved87jmfc294n8/5TDHGoJRSKvLFhDsDSimlQkMDulJKRQkN6EopFSU0oCulVJTQgK6UUlGiRrhO3KRJE5OcnByu0yulVET68ccfDxljEq32hS2gJycnk56eHq7TK6VURBKR3b72aZWLUkpFCQ3oSikVJTSgK6VUlAhbHbpSSoVCYWEh2dnZFBQUhDsrIRUfH0+rVq2Ii4sL+jMa0JVSES07O5uEhASSk5MRkXBnJySMMeTn55OdnU3btm2D/pxWuSilIlpBQQGNGzeOmmAOICI0bty4zE8dGtCVUhEvmoK5w/lcU8QFdGMM/0nfy9mi4nBnRSmlqpSIC+gLNx9g/Mcb+fviHeHOilJKAVCvXr1wZwGIwIB+7EwhAPknz4U5J0opVbVEXEBXSqmqyhjD+PHj6dy5M126dGH27NkA7N+/n379+tG9e3c6d+7M8uXLKS4u5s4773Sm/dvf/lbu82u3RaVU1Hjyiwy27Dse0mOmtqzPE9d1Cirtp59+yvr169mwYQOHDh2iV69e9OvXjw8//JDBgwczceJEiouLOX36NOvXrycnJ4fNmzcDcPTo0XLnNeJK6LoCqlKqqvruu++45ZZbiI2NpVmzZvTv3581a9bQq1cv3nrrLSZPnsymTZtISEigXbt27Nq1iwceeICFCxdSv379cp8/YkvoUdhLSSlVTsGWpCuKMdZFzn79+rFs2TLmzZvHb37zG8aPH8/tt9/Ohg0bWLRoETNnzmTOnDn8+9//Ltf5I66ErpRSVVW/fv2YPXs2xcXF5OXlsWzZMnr37s3u3btp2rQp99xzD2PGjGHt2rUcOnSIkpISbrjhBp566inWrl1b7vNHbAldKaWqmuuvv56VK1fSrVs3RITp06fTvHlz3nnnHWbMmEFcXBz16tXj3XffJScnh7vuuouSkhIAnnnmmXKfP+ICuo8nGqWUCpuTJ08CttGdM2bMYMaMGW7777jjDu644w6vz4WiVO4q4qpcjL1ZVOvQlVLKXcQF9FIa0ZVSylUEB3SllLLx1bskkp3PNWlAV0pFtPj4ePLz86MqqDvmQ4+Pjy/T57RRVCkV0Vq1akV2djZ5eXnhzkpIOVYsKougArqIDAH+DsQCbxhjpnnsvxB4B2hoTzPBGDO/TDkpI20UVUoBxMXFlWlVn2gWsMpFRGKBmcBQIBW4RURSPZL9BZhjjOkBjAZeCXVGlVJK+RdMHXpvINMYs8sYcw6YBYzySGMAx0QEDYB9ocuiUkqpYAQT0JOAvS7vs+3bXE0GbhORbGA+8IDVgURkrIiki0j6+dZ3aRW6UkpZCyagW9VWe8bVW4C3jTGtgGHAeyLidWxjzGvGmDRjTFpiYmLZc2s7iM9MKaVUdRZMQM8GWru8b4V3lcoYYA6AMWYlEA80CUUGfdGSulJKuQsmoK8BUkSkrYjUxNboOdcjzR7gFwAicgm2gF4hfYgOHj8LwIer91TE4ZVSKmIFDOjGmCJgHLAI2IqtN0uGiEwRkZH2ZI8A94jIBuAj4E5TQb38DxwvqIjDKqVUxAuqH7q9T/l8j22TXF5vAfqENmu+8lIZZ1FKqcgTcUP/jdaeK6WUpYgL6EoppaxpQFdKqSgReQFda1yUUspSxAX0Em0VVUopSxEX0Is1niullKWIC+hfbNB5v5RSykrEBXSllFLWNKArpVSU0ICulFJRQgO6UkpFCQ3oSikVJTSgK6VUlNCArpRSUUIDulJKRQkN6EopFSU0oCulVJTQgK6UUlFCA7pSSkWJoAK6iAwRke0ikikiEyz2/01E1tv/+0lEjoY+q0oppfwJuEi0iMQCM4FBQDawRkTm2heGBsAY83uX9A8APSogr0oppfwIpoTeG8g0xuwyxpwDZgGj/KS/BfgoFJlTSikVvGACehKw1+V9tn2bFxFpA7QFvvaxf6yIpItIel5eXlnz6mXf0TPlPoZSSkWLYAK6WGzztW7QaOBjY0yx1U5jzGvGmDRjTFpiYmKwefRpc86xch9DKaWiRTABPRto7fK+FeBr2aDRaHWLUkqFRTABfQ2QIiJtRaQmtqA91zORiHQEGgErQ5tFpZRSwQgY0I0xRcA4YBGwFZhjjMkQkSkiMtIl6S3ALGNMpS3j/OEPezh5tqiyTqeUUlVawG6LAMaY+cB8j22TPN5PDl22gvPN9jwmz83guZu6VfaplVKqyon4kaJHTp0LdxaUUqpKiPiArpRSyibiA7pYdapUSqlqKOID+rniSmuDVUqpKi3iA/qyn8o/4lQppaJBxAd0pZRSNlER0F9ftivcWVBKqbCLioA+df7WcGdBKaXCLioCulJKKQ3oSikVNTSgK6VUlNCArpRSUUIDulJKRQkN6EopFSU0oCulVJTQgK6UUlEiagJ6JS6UpJRSVVLUBPR3V+4OdxaUUiqsggroIjJERLaLSKaITPCR5mYR2SIiGSLyYWizGdjW/ccr+5RKKVWlBFxTVERigZnAICAbWCMic40xW1zSpACPAX2MMUdEpGlFZdgXrXFRSlV3wZTQewOZxphdxphzwCxglEeae4CZxpgjAMaY3NBmMzCDRnSlVPUWTEBPAva6vM+2b3PVAeggIt+LyCoRGWJ1IBEZKyLpIpKel6cLUyilVCgFE9CtVu30LA7XAFKAAcAtwBsi0tDrQ8a8ZoxJM8akJSYmljWvfhXpUnRKqWoumICeDbR2ed8K2GeR5nNjTKEx5mdgO7YAX2m+2nqwMk+nlFJVTjABfQ2QIiJtRaQmMBqY65Hmv8DVACLSBFsVTOUuI6QFdKVUNRcwoBtjioBxwCJgKzDHGJMhIlNEZKQ92SIgX0S2AEuB8caY/IrKtJUTZ4v408cbOXa6sDJPq5RSVUbAbosAxpj5wHyPbZNcXhvgD/b/wmZ2+l7q1qrBpOtSw5kNpZQKi4gbKdq77QV+92v3RaVUdRVxAf3ZG7qGOwtKKVUlRVxAr1fLfy3RV1sO0n/GUs4WFVdSjpRSqmqIuIBes4b/LGcfOcPu/NMcOFZQSTlSSqmqIeICeoPacUGl07ldlFLVTcQF9GBpPFdKVTdRG9CVUqq60YCulFJRImoDui5Jp5SqbqI2oCulVHUTtQFdy+dKqeomegO6VrkopaqZqA3oA19YFu4sKKVUpYragK6UUtVNVAf0vBNnw50FpZSqNFEd0HtNXRzuLCilVKWJ6oCulFLVSdQH9H99u5MVmYfCnQ2llKpwQS1BF8meWbANgKxpw8OcE6WUqlhBldBFZIiIbBeRTBGZYLH/ThHJE5H19v/uDn1WlVJK+ROwhC4iscBMYBCQDawRkbnGmC0eSWcbY8ZVQB6VUkoFIZgSem8g0xizyxhzDpgFjKrYbCmllCqrYAJ6ErDX5X22fZunG0Rko4h8LCKtrQ4kImNFJF1E0vPy8s4ju0oppXwJJqCLxTbPiVK+AJKNMV2BxcA7VgcyxrxmjEkzxqQlJiaWLadKKaX8CiagZwOuJe5WwD7XBMaYfGOMY1jm68ClocmeUkqpYAUT0NcAKSLSVkRqAqOBua4JRKSFy9uRwNbQZdFbXKzVQ4NSSlVvAQO6MaYIGAcswhao5xhjMkRkioiMtCd7UEQyRGQD8CBwZ0VlGOAPgzpW5OGVUioiBTWwyBgzH5jvsW2Sy+vHgMdCmzXf6tSMraxTKaVUxIjIof+6eIVSSnmLyIB+vuas2cv6vUfDnQ2llKoQUT+Xi8PiLQd59JONALxxexrZR05zZ5+2Yc6VUkqFTrUJ6C8u+cn5+u530wE0oCulokpEVrmcTw365pzjIc+HUkpVJREZ0EPls3XZ4c6CUkqFTLUO6L+fvSHcWVBKqZCp1gHd1cLNB0ieMI9H5miQV0pFpogM6CO6tqRdYt2QHOvY6UIA7n3/RwA+WavVMEqpyBSRAT0xoRZfPzIgJMd6bfnOkBxHKaXCLSIDeijN33SAXXknw50NpZQqt2of0H8+dIprnv823NlQSqlyi+iA3qpR7XBnQSmlqoyIDuh/GZ4a7iwopVSVEdEBvWaNilnoYu/h0xVyXKWUqkgRHdDFcrnT8us7fSlzN+yz3FdUXBJw+t59R8+wJ19vCkqpyhXRAb0ivbRkB7knCty2FRQWc9HEBTz/5U8+PmVz5bSv6TdjaUVmTymlvER2QK/ApUV35J7kpldXum07ebYIgI9+2FNxJ1ZKqfMU0QG9opeK3u1RbbJ0Wy4A+afOcaKgsILPrpRSZRNUQBeRISKyXUQyRWSCn3Q3iogRkbTQZTH8Zi7NZOm2XBZsPuDcNvq1VWHMkVJKeQu4wIWIxAIzgUFANrBGROYaY7Z4pEsAHgRWV0RGfeStws+RPGGe5faMfTq/ulKqagmmhN4byDTG7DLGnANmAaMs0j0FTAcKLPZViB4XNiQxoRZ1a8ZW1indnLLXqSulVFUQTEBPAva6vM+2b3MSkR5Aa2PM//wdSETGiki6iKTn5eWVObOe6sfHsWbiQHq2aVTuY52P577cHpbzKqWUlWACulW9hrMjtojEAH8DHgl0IGPMa8aYNGNMWmJiYvC5rKLOnCsOdxaUUsopmICeDbR2ed8KcB11kwB0Br4RkSzgcmButDWMWnGMLzpzrtg5r3pZ7D92JuAgJaWUClYwAX0NkCIibUWkJjAamOvYaYw5ZoxpYoxJNsYkA6uAkcaY9ArJcRU0+MVldJvypc8GVCs/7j7CFc98zcc/6oIaSqnQCBjQjTFFwDhgEbAVmGOMyRCRKSIysqIzWJUZDM8s2MqeMs79UlRcwro9RwBbYFdKqVAI2G0RwBgzH5jvsW2Sj7QDyp+tyDB3wz4KCkvK/LlH/rOBz9fbaq3OFpX980opZSWiR4qGm1Uw//nQKefrvYdPk+Xy3sERzAGOnj5XMZlTSlU7GtBD7OrnvnG+7jt9KQNc3ls5V1zCB6t3M+iFb0meMI9zWmJXSp2noKpcqrrKGDFaUb7PzOf7zHzn+9wTBbRqVCeMOVJKRSotoVcx1zxXtvVN527Yx5FTWm2jlNKAXiky9h0LOu25Yu8ql805x9h/7IzX9pyjZ3jwo3Xc+/6P5cqfUio6aECvBMP/8R3/Sd/L2HfTSZ20sMyfH/HSd1zxzNfO9+lZh7n+le85WWCbS2b1z4ed+04UFDLxs01BjWJdt+cIuccrbeodpVQFi4qA7qhBf+uuXmHNhz/jP97Il1sOcjoE0wU89ukm1u05Sla+dw+amUt38sHqPby3Kivgca5/ZQWD/ras3PlRSlUNUdEoGm0GvfAtO3JPsuSR/rRPrOczndWsASX2jf5mFPhme65zIexjZ/xPWXDmXDG1asQQExO5Dc9KVRca0KugHbknAVi85SDt+5cG9BMFhbyzIsu5f/x/Njj3JU+Yxws3d3O+9zdDzJ1vrQkqH6fOFtHpiUWIwM6pwzSoK1XFRUWVy8DUZgC0bVw3zDkJLQNuC1V3mfwlz7ksUH3CYz72z9fv87ss37YDx/nnNzu9tr/w5XbeX7Xba/tx+zJ7xsBXWw8GlefXl+3im+22pfqyDp3SyceUqkRRUUK/7bILGdW9JfXj48KdlZCatmAb0xZsCzq9a+jce/g0MxZt4/CpQp75VRcArnvpOwqLvQPsP77OBOC2y9u4H88labADnqbO3wrA3HF9GPny9zw+IpUxV7XlXFEJMQI1YqOiDKFUlRQV/7pEJOqC+flwLQ1/sHoPM5fu5KMf9gC2unCrYO7q07XuMz+Wp2ztWGB7rX0Ssg5/WcCIl74rxxGVUoFERUBXNit35nPopPUgo0uC6C75hzmldfKz1+xhwicby58pl7vCtgMnvHZv3X/cWbWjlCofDehRpKjE8Mla7/nVf3Dppx7I9gMn2JxzjD99sonlOw4F9ZmMfd4Dn4KdjWHo35dz6+uB1xU/evocxSVaHx8qOw6eIHnCPHZbdH1VkSvqAvovu7cMdxaqnJv/tTLotINfXGZZNZJ34qzPzwz/h/vAJyitf5+3aX/Ac27K8T+S9tjpQrpP+Yrpi4JvT1D+fWy/8c/fdCDMOVGhFHUB/cXRPcKdhaj0zIKtXtveWZFVppuFp1MevXR8OXrGVo20QINP6OjDTlSKuoDuKi5W+02HSmGx8Zre94m5GT6rc4Kpcnlo1jrn6/Ssw2zZdxyAXXknAw54spJz9IzOL19GETxRqbIQlQG9frytN+bU67uEOSfR57N1tkf1HQe9Gzhd7T1cWqd++79/cL5ekVlaL792z1Hn6xtfXcmwfywH4Jrnv2Xky97VPo6l/nJPFPDy1zu8+rj3mfY1/aYvDfZSlIo6QQV0ERkiIttFJFNEJljsv1dENonIehH5TkRSQ5/V4MXZ+zpf3bFpOLMRlf70ySa+zDgQcA6YZxeW1ncv+ynP+TrXpS7+sJ9pfx3dHq94ZonbYKoVmYf4/ez1PPflT3z0w16vzx0vKGJPfukar8YY5m/az7miEgbMWMqMMNTDr91zhPve/1EbdVWFCxjQRSQWmAkMBVKBWywC9ofGmC7GmO7AdOCFkOe0DPQxsmKNfc96ut52j80L+bn2Hyvgiw2lS/ZlHz3DqbO2Cc7+/NkmFmzaz8qd+Ww7cNyZpt+M0lL6N9vz+N0Ha3lx8U9k5Z9m5tLSkbLbDhxnc84xVu7M59VvvUfQLt+Rx5qsw9z9TnqZV5Lasu+4c/Ttfe//yILNB/w2LFc2vbVEp2BGivYGMo0xuwBEZBYwCtjiSGCMOe6Svi5h/nsZ3etCXl6aSb1aUTEQNmIEUwB9ePZ6ftkjidPnrBtEx7ztf56ZRz/eSI8LGzrf3/fBWr/pj9jr1Pcf854meMiLy93e39u/vfP18YJCfvNmaVXRppxjXNqmkd9zuXJUH912eRsOHq86gdzBUV1VXcs+ew+fpu/0pfx9dHdGdU8Kd3ZCJpgqlyTA9dk2277NjYjcLyI7sZXQHwxN9s7PI9d2YMfUodSuGRvObCgfVu3KJ3XSIst9S7blBvz8Ope6d1+KSwxPfL6Z7CO2unzX+vb3Vu3mvZVZFsc9wtb9trKJZ4nc6qnvbFGxs9R99PQ51mQdZs6avSRPKH1S8VetFGkWbj7Ay1/vCNnxLn96Cb965fuQHc9T8oR5zFi0jdwTBdz9zhq3AWyO39n16a8inDlXTHpW8ONAyiuYIqzVTdyrLGaMmQnMFJFfA38B7vA6kMhYYCzAhRdeWLacloGIOHu4NKlX0+foSRUeo19bFVQ618BYVuv3HuWdlaUTjv13fek/3Mf/u9nyM9e/ssL5+q073efWd/wjuPlfK8k+fJqpv+rCXfZZK7OmDee2N1ezOee429MDQM+nvgoqv4syDvDQrHWse/xaZ0Gkz7SvKSopYfWfB1JYXOJsG/KUe7yARRkH+M0VyUGd63w5VsYad01KSI534HgBByp4gZWZS3dysqCIxVtz+fTHbO7s0xawXoc4M/ckFzX1PV31+fjjxxuYt3E/qx77Bc0bxIf02FaCKaFnA61d3rcC/N3WZgG/tNphjHnNGJNmjElLTEwMPpfl8NXv+7Pw4b7c2789254aUinnVOG376j3kn1lcZdH1c/RM4UkT5jHDz8fZt+xAmcwd9icYyvx+avCuPyZJWQfsTXYPvbpJrceP88u2EZBYQk5LvnOOXqGg8fPMm/jflImLiAz171nUUFhMScKCrnn3XQe/zzDeeyyqIj2plNni3xWqTmU9/fx9H3mIT5fn2O5z/GUZsXx4LZ4y0EGvvCtV4n91NkiXlu2kxKP+sSSEsNzi7ZzMMANaau9K+6zC7dRaLG8ZKgFE9DXACki0lZEagKjgbmuCUTE9ZY9HAjdc1k5Napbk4ub12fC0IuJj3Ovgumb0oSBl2hPmGj0wEfrAicqgzeX/+xz35oyPFIv2WqrUvrohz38+o3SKQ8c4cIRYOe6BJaFGbYBVRn7XJuq4OLHF9Jl8pcctrcTOHrRfLM9l/yT/uvtX7dfz8+HTrPOPoGap3V7jjB5bkaZp0Du9MQiOj9hXaUGtievK6eVjix+cfFPLMoo36CxW99YzUOz1lvuc1SvuHLcxxxX5mhUd0175lwxj3++mafnb/PK37q9R3h5aSaXPb3E//djP9Fn63KYtca7V1aoBQzoxpgiYBywCNgKzDHGZIjIFBEZaU82TkQyRGQ98AcsqluqiqSGtQF4dEhH3htzGW/cUXWXrVNVx3eZvue1uenV0tGyawPU7xtjvALAoZNnnXOqOALNnz4unRjNsQrVQ7PWWw64ctT3nz5XzPurdnPnW2vcGnRdHS8oZOn20naKj37Y46xq2ph9lKxDpXO7XP/KCt5ekeW1+tUKi+9iydaDJE+YR0FhsT3P8Nb3P7NqVz4bs/1/Jy8u3sH/+eg55c+ps0Us35HHxM82Obf9uNv65uTJceP82qPNxrE9Peswl0xayKdrbaX+sx5tKq4F9sVb3Y/hq0fUN9tyfT5FhEpQ3UCMMfOB+R7bJrm8fijE+aow8x/qy/EzhbS+oE64s6KqoclfbHErqZ06W0TaXxc736/dc5R2ifXcqkLmbSydD6fbk1/y4d2Xsd1iYNdT/9vCip35AOzMO2l5/odnrfcKYg4jX7Y1UGZNG+623bNa5tdvrCZr2nCOnDrHsh15jOqexJh30gHY7DIvz5NfODvC0a11Qz6/v4/leR2+2nKQr7flOufvdxyvaf1aNE0orX8uLC5h1a58Pli1x/n04nDDP1fw9l29SG1R3+v4U/63hQPHz/KnIR29rslx0xL7LfXGV92ntFj2Ux5DuzRn9pq93HpZG7eqtcOnSp+G5m/az+8+WMuih/vRsXmCW7ol23JZsi23QnvVVLt+fQ1qx9Ggtvvc6aN7ta6UxyGlwH0a4U4eVRN//M8GSozxu5i4a1UNlAYhRzC3Mm/jfu7/cC1NE2pZ7v9qi/uKVHfbA7TDhr3epezffbCWlbvy3bpz+pqyYcPeoyRPmMclFoHW4Z53befsnFSfLkkN6NSygXOiOMdN5sy5YqbO38L7q/b4PM5976/lTGHp97fP3mW1xMCr3+5kQMdEvsxwv15HiTvvxFky9nlPFvfpuhxW/3yYnKNnEBFSWyR4fRZKv8fNOcdsAb2SB8VUu4BuJdQt20qVx6Mfl20eel89RQoKixn34Voua9vYuZJUro/BTX+dV1qa3pV3ksUuSw7uzj/NqJnu3Qs/W5fNyl22G4jrwiljPG4Enqzqsz1N/MzWC+mVW3s6tz326Sam/rJzUPP6uwZzKw/PWu/2nY14aTlpbS4AYHb6XmanWxfuHA3WJwuKcG3+LnGpk3IN33sPn7bstvrN9lwGVNAo9qicy6Ws+neonB43SlWWs0UlzF2/j8Vbc53B3B/XaQmuef5bt30DnvvGK/1LSzLLncdAfucyaOyjH/Yw4dMQLLiC9w1wc85x3l6RFfTnM/Yd44EPS/M28bPNzvYD15tJ3+lLLQN6sIu0nw8N6EBKswQW/6FfuLOhVEiVpeeIv659VnYdqvyFMeakey/eEg7/27jfWY3j6uLHF7Jgs+07nx/EOgAVQQO63UVNE7wGhSgVyYIZdRsK3+3IC5yomgn03Ze1K2iwNKC7ePuu3s7XteNi+eaPA8KXGaUixOOfZ4Q7CxHn258q5iaojaIuXHu/bNVRpUqpChJohOn50hK6UkpVsqXbKqaErgG9jF697dJwZ0EpFeE8B0SFigb0Mkpppn3WlVJVkwb0AC5t04h6tWrwxu1pDO7UjDYWUwa8dEuPMORMKaXcaaNoAJ/cd6Xz9cDUZgA8+IsU/rHENqHkZ7+7kh4XNgr57H5KKVVWWkL3cEmL+nRr7b8/+r392wFwdcdEelzovSzZ0M7NfX721ssqbmEPpVT1piV0Dwse6hswTZ2aNZj34FW0bVLXuW3X08No92e3CSkRwWvq0anXd+GrLQd9zqmhlFLnS0vo56lTywbUqVl6P4yJEf73wFXUj6/Bo0MuZvtfh/DBmMssP/vDxIFeU5QqpVR5aUAPoc5JDdg4eTBtm9SlVo1YYmLcp86cfF1qmY/5ws3dQpU9pVSU04BegTq1dJ/72bFArcMX464KeIxGdWs6X/tbE7V32wvKmDulVLTRgF6BEuLj2DT5Wp/7u7RqQIzY5mPfNPlafjegvddNAJc6eM81UV15fU4pVe1oQK9gCfFxfvdnTh3Glw/3IyE+jkeHXEyXpAZu+0s8WlV3TB0K2FZ1cdj59DDnqjXBeu03OuJVqWgTVEAXkSEisl1EMkVkgsX+P4jIFhHZKCJLRKRN6LMauZY/erXP+dZjYsStrn3SdalMv6ErFze3LXFlDMwd14c370gDIC42hgUP9eXDey53fibWo65+xYRrALj/6vZe53MsQdalVQO6tWrgtV8pFbkCBnQRiQVmAkOBVOAWEfFs3VsHpBljugIfA9NDndFI1vqCOlzUNCFwQmxdIm/u1ZpWjWoDthqXrq0a8otLmjnTXNKiPvU9Sv6pLlUuLRvWJmvacMYPvtjr+PcNaM/6SYNo0aA2nZMCB/T5D/ble/sNQilVtQVTQu8NZBpjdhljzgGzgFGuCYwxS40xp+1vVwGtQpvN6sexuKxnlYur98dcxvQbuwJwQ88kkhrWDnxcoGGdml7b7+qTbJk+pVk9kuw3CMdNBuCai73XRLyyfeOA53fo2Cy4G5xSKnjBBPQkwHXV1Gz7Nl/GAAusdojIWBFJF5H0vDxd5cSfEV1bAP4D31UpTbg5rTVguwF8/cf+pP9loGXa9om2QVCuJX3XBcn7pSRybWozZo+9nLo1rRtfG9ezVdeMH9yR129P8wlMqSgAAA6jSURBVNrfrwxrs3ZsXvaA/u34AayfNKjMn1OquggmoFu1tlkWG0XkNiANmGG13xjzmjEmzRiTlpioCzP7M6p7EplTh5LsMho1kFo1YmliD7qeOic1IGvacFpbTC4Gtrr8125P47J2jcmYMoQaMRY/u/1p4cr2jYmNEbKmDeeVW3vy5MhO/H10d+7p286ZdEgn39MfAG6Nv0+O7BTo0gBo07iu5dOFP9r7R1UnwQz9zwZau7xvBezzTCQiA4GJQH9jjI5rD4EasaHrhBSoD4xn/H71tkt5ffkut8DuuIuLS9F+WJcWbp+rWSOGc0Ul/PX6zjSsE8esNaUPd9Nv7MqjH2+0HwPevqsXMSLUreW7O2Z5vDemN31TEkmeMO+8jxEfF0NBYYnlvut7JPHZupzzPrZSoRZMxFgDpIhIWxGpCYwG5romEJEewL+AkcaYylmZVpWJaxB2bnMJ857dHgemNmP2/13h9jlHdb6/m4NjX92aNZh2Q1d+N6A902/oyoKH+nJzWmvS2pROZjagY1P6dUh0HvdSl303XdqKerWCm2poVPeWvPvb3l7b+6YEfgpMTLB+onEY28+7p5DDBXWtnxY2TPI99sCfm9NaYfEzqSg0fnDHCjluwH8xxpgiERkHLAJigX8bYzJEZAqQboyZi62KpR7wH3sA2GOMGVkhOVbnpV+HJl7bHvjFRazdc4SOzRK4vF3gkabGXkb3F3QGd2rO3A37nF0pHx3i3tPmzTt78fS8rfzaZdbJEntAd31KmHFTN2bc1I1jpwv55qdckht7Vz09f1M3PluXw99H+5+PfsfUoRw4VsDew6dZsTOfl5dmApDaoj4f3H0ZPZ76yudn/zCoA9emNuON5bto26Qef1v8k3PfHVck8+Z3P7ulT4ivQYM6/sce+HLnlW2Zen0XUiZaNkFVmOFdWzBv4/5KPWd1F6ggcb6CKgIZY+YD8z22TXJ5bd0Sp8Ju/aRB5J04S4pF42rThHjmPRh4dsmyeO6mbjw27GJq1rB++GtQO45n7T1zHBw9eQTxGljVoE4co7q7t8G/eltPLmpaj4uaJnDDpd4dqsb2a+d204mLjaH1BXVofUEdOjZPcAb0F0d3d5taITZGKC7xbh7qnNSAF+03jbyTBby/ag+Pj0jlwsZ1+Pz+Poya+b2vr6NMkhrVJi42huk3dqVbq4ZsP3iCnCNneHbhtpAc38pdfZL505CLNaBXMss2qhDQkaJRrmGdmpbB/Hzc1/8iAL8NtTVrxNCiQeDuk66cAV3giweu4osH/M9xM6RzC7/9+v887BIeG3pJwPN2sH8vzevHA7YRt6/fnkbvZN9PK4+PSGX84I7cfoVt7Fy31g1p5Foit98P7r6qrcWn/asfbytf3ZzWmo7NExjZrSX3WU0HEaQuSQ2YPfZyv2luu9z3GMC+Kd5Pdf4886suztdZ04aT3Ni6AT6UgpnDKNB3EA6pFdRYrwFdBW141xZkTRvuNaipvGrb56hp0SC+XMeZfF0qlwX4B27VlrD4kf6smWh7yByU2ow5917h8/O1asRy/9UXEefSYG01QOsvI1JZ97h1F0ur8QKOG4SVicN835zaNK7D9Bu6em3/5L4r+eS+K6lf2/u3uqRFaTBx/TYa1omjx4Wli7u8etulzB3Xx+vzt/S2VZc9cM1Fzm1Pjuzk3O7wzfiryZo23O+11XJ5khvo0qX2//q3s0rupn+HRJ6/qXQ20ocHprjtH9AxkasuakJ3+zU1rx/Pszd0IZD/l9Y6YJpaFk+gAzqWodtuBY3D0AUuVNj1uLARz9/UjcF+VnoKxp192nrNaOlLQ5dSdb1aNbwaYGf+uidN6gXXRfKVW3uyYe8xbntztVvJq569xH1v//YMSm3KDf9cCUCN2NIw+s5ve7Nw836mjOrs8/hN/NS3fjv+agAe/WSj23ZHA7PVuLQbeibx13nHAdsNLj4ulsdHpHJ1x0SSG9el3Z/n88drO1C3Vg26tvJeveuZX3VxlsYfuTZw416Mn0YX110v3dKDSyYtBGBEl5b869tdzn0XNa1HZu5J3v1tb3bmneTJL7YAtlHYm58cTO24WF76eofbsQd0SHT+PWyZMphaNWJZuq20z8bvB3ZwaxOZO64PJcYWbFs2rM2pc0W8tmwXVi5v15hvf/I9lmbDpGvpNuVLn/srigZ0VSVY1YVXhGBrLod3bRE4kV1CfBxXpTTh43uvoIPLgKm42BjLhUx6J19AvVo1mDKqM5e2aUT/AAOyOjRL4JP7rmTGom2s2nWYxnVrkn/qnFua8YM7MmPRdq/PWjW+jbmqLc0bxPPK0p3Op4UxLlVEvhZfSahVw20OoWD5a0SvVyuOgsKz/KpHErVrxtI5qT6bc457pXvj9jRmrdlL35QmFHvcpRw34wEdm/LiYltQf/nXPRjaufQ3dCxG43iaev32NAalNmN41+YMfGEZgNvN66GBKbxoD/aje7Xmj4M7kvbXxYDtqeiVW3uSlX+K4f/4zvK6fDWM9257AW/ckWb5pBgKGtBVtdKgdhxDOzfnriBL8mWR5qfu3VV8XGyZG6MvbdOIWWNLq4I8+9bf1789xhgGpTYn/2TpMBDXgH59jySW7ziEiDCia0tGdG1Zpjw0qBNHl/OY0M21hJ7cuA5Z+adp0SCe/ccKaJpQi7fv6sVFTet5fW5Ip+YszDjApBGpJDepy4Shth5TjraPX/Zwz3/31g356vf92Jh9zOe1NW8Q73bDuqhpAq/fnkb2kdNeae/u247cE2eZOOwS6tqf4k6eLQKgbq0adGrp/l0EE6Ln/J/v6rxQ0ICuqpWYGOGft4Vn6uA7r0zm7RVZ1PExtUJZufYIiokRxl3jqEN2r58dc1Vb+qY0YUBH7/l3KsKFHqORH7wmxdm9c8qozvTrkMjmnGOMeOk7GtSO8zlJ3Mxbe1JYXOK1DoBjbiErKc0SytwJYFBqM8vt9WrV4OnrS+vcP7rncq57+Tu3wP3nYRfz9HxbL6QHfpHC0u2l1TDzHryKrftP8MbyXUwZ1ZkUi5tWqGlAV6qS/GX4JTSpV5MxVwVu8AtkwxPXWjbMWXl8RNmXPnTVPrEuO/NOcVnbwJOvfXj3ZV4BtUGdOPqmNGH5jkPOHk2dWtbnietSvUrSfxpyMb+fvZ72TesSGyPExlTMKOLzkdzEdqNybfwd268932zPY8XOfHpe2MgtfaeWDejUsgE3VlJ1IoAYP7P5VaS0tDSTnp4elnMrpYJ3tqiYl7/O5A+DOpx33e/qXfnc9uZqfvjzQLe+/9GgoLCYEwVFJCbUclaFVeQi8CLyozHGe3Y8tISulAqgVo3YoHqz+HNZu8bsmDosRDmqWuLjYv0uD1mZNKArpVSI/Pf+PmTsOxa282tAV0qpEOneuiHdW3v33a8sOlJUKaWihAZ0pZSKEhrQlVIqSmhAV0qpKKEBXSmlooQGdKWUihIa0JVSKkpoQFdKqSgRtrlcRCQP2H2eH28CHAphdqqq6nCd1eEaoXpcp15j5WhjjLGcRD9sAb08RCTd1+Q00aQ6XGd1uEaoHtep1xh+WuWilFJRQgO6UkpFiUgN6K+FOwOVpDpcZ3W4Rqge16nXGGYRWYeulFLKW6SW0JVSSnnQgK6UUlEi4gK6iAwRke0ikikiE8Kdn7ISkSwR2SQi60Uk3b7tAhH5SkR22P/fyL5dROQf9mvdKCI9XY5zhz39DhG5I1zX45Kff4tIrohsdtkWsusSkUvt31um/bPnt7hlOfi4xskikmP/PdeLyDCXfY/Z87tdRAa7bLf8GxaRtiKy2n7ts0Wk0hffFJHWIrJURLaKSIaIPGTfHjW/pZ9rjPzf0hgTMf8BscBOoB1QE9gApIY7X2W8hiygice26cAE++sJwLP218OABYAAlwOr7dsvAHbZ/9/I/rpRmK+rH9AT2FwR1wX8AFxh/8wCYGgVucbJwB8t0qba/z5rAW3tf7ex/v6GgTnAaPvrV4H7wnCNLYCe9tcJwE/2a4ma39LPNUb8bxlpJfTeQKYxZpcx5hwwCxgV5jyFwijgHfvrd4Bfumx/19isAhqKSAtgMPCVMeawMeYI8BUwpLIz7coYsww47LE5JNdl31ffGLPS2P6FvOtyrErj4xp9GQXMMsacNcb8DGRi+/u1/Bu2l1KvAT62f971+6o0xpj9xpi19tcngK1AElH0W/q5Rl8i5reMtICeBOx1eZ+N/x+iKjLAlyLyo4iMtW9rZozZD7Y/NqCpfbuv642U7yFU15Vkf+25vaoYZ69u+LejKoKyX2Nj4Kgxpshje9iISDLQA1hNlP6WHtcIEf5bRlpAt6pri7R+l32MMT2BocD9ItLPT1pf1xvp30NZr6sqX+8/gfZAd2A/8Lx9e0Rfo4jUAz4BHjbGHPeX1GJbRFynxTVG/G8ZaQE9G2jt8r4VsC9MeTkvxph99v/nAp9he2w7aH8Uxf7/XHtyX9cbKd9DqK4r2/7ac3vYGWMOGmOKjTElwOvYfk8o+zUewlZdUcNje6UTkThsge4DY8yn9s1R9VtaXWM0/JaRFtDXACn2FuSawGhgbpjzFDQRqSsiCY7XwLXAZmzX4OgFcAfwuf31XOB2e0+Cy4Fj9sfdRcC1ItLI/lh4rX1bVROS67LvOyEil9vrJ293OVZYOYKc3fXYfk+wXeNoEaklIm2BFGyNgZZ/w/b65KXAjfbPu35flcb+/b4JbDXGvOCyK2p+S1/XGBW/ZWW0vIbyP2yt6j9ha12eGO78lDHv7bC1hG8AMhz5x1bntgTYYf//BfbtAsy0X+smIM3lWL/F1jiTCdxVBa7tI2yPqYXYSi5jQnldQBq2f2A7gZexj3KuAtf4nv0aNmL7h9/CJf1Ee36349KTw9ffsP3v4wf7tf8HqBWGa7wKW/XARmC9/b9h0fRb+rnGiP8tdei/UkpFiUirclFKKeWDBnSllIoSGtCVUipKaEBXSqkooQFdKaWihAZ0pZSKEhrQlVIqSvx/ZN3SqV21/H0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-08ed19143c33>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions_logp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mactual_next_tokens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mscheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Soft/anaconda3/envs/ml/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Soft/anaconda3/envs/ml/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = model.cuda()\n",
    "model.train()\n",
    "\n",
    "for i in range(50000):\n",
    "    batch_ix = to_matrix(random.sample(texts, 128), token_to_id, max_len=MAX_LEN)\n",
    "    batch_ix = torch.tensor(batch_ix, dtype=torch.int64).cuda()\n",
    "    \n",
    "    logp_seq, hid = model(batch_ix)\n",
    "    \n",
    "    predictions_logp = logp_seq[:, :-1]\n",
    "    actual_next_tokens = batch_ix[:, 1:]\n",
    "\n",
    "    loss = -torch.mean(torch.gather(predictions_logp, dim=2, index=actual_next_tokens[:,:,None]))\n",
    "    \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    scheduler.step()\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    history.append(loss.data.cpu().numpy())\n",
    "    if (i + 1) % 100 == 0:\n",
    "        clear_output(True)\n",
    "        plt.plot(history[100:], label='loss')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "        \n",
    "    if (i + 1) % 5000 == 0:\n",
    "        torch.save(model.state_dict(), f'model{len(history)}.lol')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Сохранение весов модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'model-final.lol')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}