From 6dd9ae15c3e5b3bdc7f458977404a91b401845eb Mon Sep 17 00:00:00 2001 From: nnnyt <793313994@qq.com> Date: Mon, 1 Mar 2021 21:26:22 +0800 Subject: [PATCH] add scripts --- scripts/dataset/assistment.ipynb | 745 +++++++++++++++++++++++++++++++ scripts/test.ipynb | 367 +++++++++++++++ scripts/train.ipynb | 406 +++++++++++++++++ 3 files changed, 1518 insertions(+) create mode 100644 scripts/dataset/assistment.ipynb create mode 100644 scripts/test.ipynb create mode 100644 scripts/train.ipynb diff --git a/scripts/dataset/assistment.ipynb b/scripts/dataset/assistment.ipynb new file mode 100644 index 0000000..acfec2d --- /dev/null +++ b/scripts/dataset/assistment.ipynb @@ -0,0 +1,745 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "import json\n", + "import random\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from collections import defaultdict" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def stat_unique(data: pd.DataFrame, key):\n", + " if key is None:\n", + " print('Total length: {}'.format(len(data)))\n", + " elif isinstance(key, str):\n", + " print('Number of unique {}: {}'.format(key, len(data[key].unique())))\n", + " elif isinstance(key, list):\n", + " print('Number of unique [{}]: {}'.format(','.join(key), len(data.drop_duplicates(key, keep='first'))))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/yutingning/opt/anaconda3/envs/cat/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3147: DtypeWarning: Columns (18) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " interactivity=interactivity, compiler=compiler, result=result)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0order_idassignment_iduser_idassistment_idproblem_idoriginalcorrectattempt_countms_first_response...hint_counthint_totaloverlap_timetemplate_idanswer_idanswer_textfirst_actionbottom_hintopportunityopportunity_original
013302253727761864525331395142411132454...033245430799NaN260NaN11.0
12330227092776186452533150514351114922...03492230799NaN550NaN22.0
233545020422067470363331595144410225390...034200030799NaN880NaN11.0
34354502952206747036333110513951114859...03485930059NaN410NaN22.0
4535450311220674703633319651481101419813...3412456430060NaN6500.033.0
\n", + "

5 rows × 31 columns

\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 order_id assignment_id user_id assistment_id problem_id \\\n", + "0 1 33022537 277618 64525 33139 51424 \n", + "1 2 33022709 277618 64525 33150 51435 \n", + "2 3 35450204 220674 70363 33159 51444 \n", + "3 4 35450295 220674 70363 33110 51395 \n", + "4 5 35450311 220674 70363 33196 51481 \n", + "\n", + " original correct attempt_count ms_first_response ... hint_count \\\n", + "0 1 1 1 32454 ... 0 \n", + "1 1 1 1 4922 ... 0 \n", + "2 1 0 2 25390 ... 0 \n", + "3 1 1 1 4859 ... 0 \n", + "4 1 0 14 19813 ... 3 \n", + "\n", + " hint_total overlap_time template_id answer_id answer_text first_action \\\n", + "0 3 32454 30799 NaN 26 0 \n", + "1 3 4922 30799 NaN 55 0 \n", + "2 3 42000 30799 NaN 88 0 \n", + "3 3 4859 30059 NaN 41 0 \n", + "4 4 124564 30060 NaN 65 0 \n", + "\n", + " bottom_hint opportunity opportunity_original \n", + "0 NaN 1 1.0 \n", + "1 NaN 2 2.0 \n", + "2 NaN 1 1.0 \n", + "3 NaN 2 2.0 \n", + "4 0.0 3 3.0 \n", + "\n", + "[5 rows x 31 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "raw_data = pd.read_csv('../../data/assistment.csv', encoding = 'utf-8', dtype={'skill_id': str})\n", + "raw_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "raw_data = raw_data.rename(columns={'user_id': 'student_id',\n", + " 'problem_id': 'question_id',\n", + " 'skill_id': 'knowledge_id',\n", + " 'skill_name': 'knowledge_name',\n", + " })\n", + "all_data = raw_data.loc[:, ['student_id', 'question_id', 'knowledge_id', 'knowledge_name', 'correct']].dropna()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total length: 274590\n", + "Number of unique [student_id,question_id]: 270478\n", + "Number of unique student_id: 4151\n", + "Number of unique question_id: 16891\n", + "Number of unique knowledge_id: 138\n" + ] + } + ], + "source": [ + "stat_unique(all_data, None)\n", + "stat_unique(all_data, ['student_id', 'question_id'])\n", + "stat_unique(all_data, 'student_id')\n", + "stat_unique(all_data, 'question_id')\n", + "stat_unique(all_data, 'knowledge_id')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Filter data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "selected_data = all_data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "filter 15924 questions\n" + ] + } + ], + "source": [ + "# filter questions\n", + "n_students = selected_data.groupby('question_id')['student_id'].count()\n", + "question_filter = n_students[n_students < 50].index.tolist()\n", + "print(f'filter {len(question_filter)} questions')\n", + "selected_data = selected_data[~selected_data['question_id'].isin(question_filter)]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "filter 1749 students\n" + ] + } + ], + "source": [ + "# filter students\n", + "n_questions = selected_data.groupby('student_id')['question_id'].count()\n", + "student_filter = n_questions[n_questions < 10].index.tolist()\n", + "print(f'filter {len(student_filter)} students')\n", + "selected_data = selected_data[~selected_data['student_id'].isin(student_filter)]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# get question to knowledge map\n", + "q2k = {}\n", + "table = selected_data.loc[:, ['question_id', 'knowledge_id']].drop_duplicates()\n", + "for i, row in table.iterrows():\n", + " q = row['question_id']\n", + " q2k[q] = set(map(int, str(row['knowledge_id']).split('_')))\n", + " \n", + "# get knowledge to question map\n", + "k2q = {}\n", + "for q, ks in q2k.items():\n", + " for k in ks:\n", + " k2q.setdefault(k, set())\n", + " k2q[k].add(q)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "filter 10 knowledges\n" + ] + } + ], + "source": [ + "# filter knowledges\n", + "selected_knowledges = { k for k, q in k2q.items() if len(q) >= 10}\n", + "print(f'filter {len(k2q) - len(selected_knowledges)} knowledges')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# update maps\n", + "q2k = {q : ks for q, ks in q2k.items() if ks & selected_knowledges}\n", + "k2q = {}\n", + "for q, ks in q2k.items():\n", + " for k in ks:\n", + " k2q.setdefault(k, set())\n", + " k2q[k].add(q)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# update data\n", + "selected_data = selected_data[selected_data.apply(lambda x: x['question_id'] in q2k, axis=1)]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# renumber the students\n", + "s2n = {}\n", + "cnt = 0\n", + "for i, row in selected_data.iterrows():\n", + " if row.student_id not in s2n:\n", + " s2n[row.student_id] = cnt\n", + " cnt += 1\n", + "selected_data.loc[:, 'student_id'] = selected_data.loc[:, 'student_id'].apply(lambda x: s2n[x])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# renumber the questions\n", + "q2n = {}\n", + "cnt = 0\n", + "for i, row in selected_data.iterrows():\n", + " if row.question_id not in q2n:\n", + " q2n[row.question_id] = cnt\n", + " cnt += 1\n", + "selected_data.loc[:, 'question_id'] = selected_data.loc[:, 'question_id'].apply(lambda x: q2n[x])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# renumber the knowledges\n", + "k2n = {}\n", + "cnt = 0\n", + "for i, row in selected_data.iterrows():\n", + " for k in str(row.knowledge_id).split('_'):\n", + " if int(k) not in k2n:\n", + " k2n[int(k)] = cnt\n", + " cnt += 1\n", + "selected_data.loc[:, 'knowledge_id'] = selected_data.loc[:, 'knowledge_id'].apply(lambda x: '_'.join(map(lambda y: str(k2n[int(y)]), str(x).split('_'))))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total length: 61860\n", + "Number of unique [student_id,question_id]: 59500\n", + "Number of unique student_id: 1505\n", + "Number of unique question_id: 932\n", + "Number of unique knowledge_id: 22\n", + "Average #questions per knowledge: 44.38095238095238\n" + ] + } + ], + "source": [ + "stat_unique(selected_data, None)\n", + "stat_unique(selected_data, ['student_id', 'question_id'])\n", + "stat_unique(selected_data, 'student_id')\n", + "stat_unique(selected_data, 'question_id')\n", + "stat_unique(selected_data, 'knowledge_id')\n", + "print('Average #questions per knowledge: {}'.format((len(q2k) / len(k2q))))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# save selected data\n", + "selected_data.to_csv('selected_data.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# save concept map\n", + "q2k = {}\n", + "table = selected_data.loc[:, ['question_id', 'knowledge_id']].drop_duplicates()\n", + "for i, row in table.iterrows():\n", + " q = str(row['question_id'])\n", + " q2k[q] = list(map(int, str(row['knowledge_id']).split('_')))\n", + "with open('concept_map.json', 'w') as f:\n", + " json.dump(q2k, f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## parse data" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def parse_data(data):\n", + " \"\"\" \n", + "\n", + " Args:\n", + " data: list of triplets (sid, qid, score)\n", + " \n", + " Returns:\n", + " student based datasets: defaultdict {sid: {qid: score}}\n", + " question based datasets: defaultdict {qid: {sid: score}}\n", + " \"\"\"\n", + " stu_data = defaultdict(lambda: defaultdict(dict))\n", + " ques_data = defaultdict(lambda: defaultdict(dict))\n", + " for i, row in data.iterrows():\n", + " sid = row.student_id\n", + " qid = row.question_id\n", + " correct = row.correct\n", + " stu_data[sid][qid] = correct\n", + " ques_data[qid][sid] = correct\n", + " return stu_data, ques_data" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "data = []\n", + "for i, row in selected_data.iterrows():\n", + " data.append([row.student_id, row.question_id, row.correct])" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "stu_data, ques_data = parse_data(selected_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "test_size = 0.2\n", + "least_test_length=150" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "n_students = len(stu_data)\n", + "if isinstance(test_size, float):\n", + " test_size = int(n_students * test_size)\n", + "train_size = n_students - test_size\n", + "assert(train_size > 0 and test_size > 0)\n", + "\n", + "students = list(range(n_students))\n", + "random.shuffle(students)\n", + "if least_test_length is not None:\n", + " student_lens = defaultdict(int)\n", + " for t in data:\n", + " student_lens[t[0]] += 1\n", + " students = [student for student in students\n", + " if student_lens[student] >= least_test_length]\n", + "test_students = set(students[:test_size])\n", + "\n", + "train_data = [record for record in data if record[0] not in test_students]\n", + "test_data = [record for record in data if record[0] in test_students]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "def renumber_student_id(data):\n", + " \"\"\"\n", + "\n", + " Args:\n", + " data: list of triplets (sid, qid, score)\n", + " \n", + " Returns:\n", + " renumbered datasets: list of triplets (sid, qid, score)\n", + " \"\"\"\n", + " student_ids = sorted(set(t[0] for t in data))\n", + " renumber_map = {sid: i for i, sid in enumerate(student_ids)}\n", + " data = [(renumber_map[t[0]], t[1], t[2]) for t in data]\n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = renumber_student_id(train_data)\n", + "test_data = renumber_student_id(test_data)\n", + "all_data = renumber_student_id(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "train records length: 51010\n", + "test records length: 10850\n", + "all records length: 61860\n" + ] + } + ], + "source": [ + "print(f'train records length: {len(train_data)}')\n", + "print(f'test records length: {len(test_data)}')\n", + "print(f'all records length: {len(all_data)}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## save data" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "def save_to_csv(data, path):\n", + " \"\"\"\n", + "\n", + " Args:\n", + " data: list of triplets (sid, qid, correct)\n", + " path: str representing saving path\n", + " \"\"\"\n", + " pd.DataFrame.from_records(sorted(data), columns=['student_id', 'question_id', 'correct']).to_csv(path, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "save_to_csv(train_data, 'train_triples.csv')\n", + "save_to_csv(test_data, 'test_triples.csv')\n", + "save_to_csv(all_data, 'triples.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "metadata = {\"num_students\": 1505, \n", + " \"num_questions\": 932,\n", + " \"num_concepts\": 22, \n", + " \"num_records\": 61860, \n", + " \"num_train_students\": 1444, \n", + " \"num_test_students\": 61}" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "with open('metadata.json', 'w') as f:\n", + " json.dump(metadata, f)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scripts/test.ipynb b/scripts/test.ipynb new file mode 100644 index 0000000..6877dcf --- /dev/null +++ b/scripts/test.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append('..')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import CAT\n", + "import json\n", + "import torch\n", + "import logging\n", + "import datetime\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from tensorboardX import SummaryWriter" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def setuplogger():\n", + " root = logging.getLogger()\n", + " root.setLevel(logging.INFO)\n", + " handler = logging.StreamHandler(sys.stdout)\n", + " handler.setLevel(logging.INFO)\n", + " formatter = logging.Formatter(\"[%(levelname)s %(asctime)s] %(message)s\")\n", + " handler.setFormatter(formatter)\n", + " root.addHandler(handler)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "setuplogger()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "seed = 0\n", + "np.random.seed(seed)\n", + "torch.manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../logs/2021-03-01-21:19/\n" + ] + } + ], + "source": [ + "# tensorboard\n", + "log_dir = f\"../logs/{datetime.datetime.now().strftime('%Y-%m-%d-%H:%M')}/\"\n", + "print(log_dir)\n", + "writer = SummaryWriter(log_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# choose dataset here\n", + "dataset = 'assistment'\n", + "# modify config here\n", + "config = {\n", + " 'learning_rate': 0.0025,\n", + " 'batch_size': 2048,\n", + " 'num_epochs': 8,\n", + " 'num_dim': 1,\n", + " 'device': 'cpu',\n", + "}\n", + "# fixed test length\n", + "test_length = 10\n", + "# choose strategies here\n", + "strategies = [CAT.strategy.MFIStrategy(), CAT.strategy.KLIStrategy()]\n", + "# modify checkpoint path here\n", + "ckpt_path = '../ckpt/checkpoint.pt'" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# read datasets\n", + "test_triplets = pd.read_csv(f'../data/{dataset}/test_triples.csv', encoding='utf-8').to_records(index=False)\n", + "concept_map = json.load(open(f'../data/{dataset}/concept_map.json', 'r'))\n", + "concept_map = {int(k):v for k,v in concept_map.items()}\n", + "metadata = json.load(open(f'../data/{dataset}/metadata.json', 'r'))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "test_data = CAT.dataset.AdapTestDataset(test_triplets, concept_map,\n", + " metadata['num_test_students'], \n", + " metadata['num_questions'], \n", + " metadata['num_concepts'])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[INFO 2021-03-01 21:20:10,230] -----------\n", + "[INFO 2021-03-01 21:20:10,231] start adaptive testing with Maximum Fisher Information Strategy strategy\n", + "[INFO 2021-03-01 21:20:10,231] Iteration 0\n", + "[INFO 2021-03-01 21:20:10,260] auc:0.6484533447389293\n", + "[INFO 2021-03-01 21:20:10,262] cov:0.0\n", + "[INFO 2021-03-01 21:20:10,262] Iteration 1\n", + "[INFO 2021-03-01 21:20:11,768] auc:0.6492874695641851\n", + "[INFO 2021-03-01 21:20:11,770] cov:0.0503951833541452\n", + "[INFO 2021-03-01 21:20:11,773] Iteration 2\n", + "[INFO 2021-03-01 21:20:13,255] auc:0.6504904035191303\n", + "[INFO 2021-03-01 21:20:13,255] cov:0.09758199115948935\n", + "[INFO 2021-03-01 21:20:13,256] Iteration 3\n", + "[INFO 2021-03-01 21:20:14,841] auc:0.6521533779951826\n", + "[INFO 2021-03-01 21:20:14,843] cov:0.13904934331124966\n", + "[INFO 2021-03-01 21:20:14,845] Iteration 4\n", + "[INFO 2021-03-01 21:20:16,413] auc:0.6533600336989795\n", + "[INFO 2021-03-01 21:20:16,414] cov:0.1810611882609775\n", + "[INFO 2021-03-01 21:20:16,415] Iteration 5\n", + "[INFO 2021-03-01 21:20:17,927] auc:0.6548746560294985\n", + "[INFO 2021-03-01 21:20:17,928] cov:0.2138639398207279\n", + "[INFO 2021-03-01 21:20:17,929] Iteration 6\n", + "[INFO 2021-03-01 21:20:19,460] auc:0.6557371632351279\n", + "[INFO 2021-03-01 21:20:19,461] cov:0.24498131873469464\n", + "[INFO 2021-03-01 21:20:19,462] Iteration 7\n", + "[INFO 2021-03-01 21:20:20,909] auc:0.6569063425461397\n", + "[INFO 2021-03-01 21:20:20,910] cov:0.2693855756697466\n", + "[INFO 2021-03-01 21:20:20,911] Iteration 8\n", + "[INFO 2021-03-01 21:20:22,342] auc:0.6578221516679326\n", + "[INFO 2021-03-01 21:20:22,343] cov:0.2909745397361497\n", + "[INFO 2021-03-01 21:20:22,344] Iteration 9\n", + "[INFO 2021-03-01 21:20:23,781] auc:0.6591130483479124\n", + "[INFO 2021-03-01 21:20:23,781] cov:0.313530251083772\n", + "[INFO 2021-03-01 21:20:23,782] Iteration 10\n", + "[INFO 2021-03-01 21:20:25,208] auc:0.6602825512892591\n", + "[INFO 2021-03-01 21:20:25,209] cov:0.3378566690173305\n", + "[INFO 2021-03-01 21:20:25,213] -----------\n", + "[INFO 2021-03-01 21:20:25,214] start adaptive testing with KL Information Strategy strategy\n", + "[INFO 2021-03-01 21:20:25,215] Iteration 0\n", + "[INFO 2021-03-01 21:20:25,233] auc:0.6434585636017105\n", + "[INFO 2021-03-01 21:20:25,233] cov:0.0\n", + "[INFO 2021-03-01 21:20:25,233] Iteration 1\n", + "[INFO 2021-03-01 21:20:31,442] auc:0.6445390034748836\n", + "[INFO 2021-03-01 21:20:31,445] cov:0.0503951833541452\n", + "[INFO 2021-03-01 21:20:31,448] Iteration 2\n", + "[INFO 2021-03-01 21:21:31,010] auc:0.6466906273936509\n", + "[INFO 2021-03-01 21:21:31,010] cov:0.09758821238631525\n", + "[INFO 2021-03-01 21:21:31,011] Iteration 3\n", + "[INFO 2021-03-01 21:22:13,886] auc:0.647910293036912\n", + "[INFO 2021-03-01 21:22:13,887] cov:0.1480900453431908\n", + "[INFO 2021-03-01 21:22:13,888] Iteration 4\n", + "[INFO 2021-03-01 21:22:47,299] auc:0.6495854036505243\n", + "[INFO 2021-03-01 21:22:47,300] cov:0.18953852469760513\n", + "[INFO 2021-03-01 21:22:47,301] Iteration 5\n", + "[INFO 2021-03-01 21:23:16,619] auc:0.6505211807639825\n", + "[INFO 2021-03-01 21:23:16,620] cov:0.22356989488930729\n", + "[INFO 2021-03-01 21:23:16,622] Iteration 6\n", + "[INFO 2021-03-01 21:23:40,716] auc:0.6521429894614312\n", + "[INFO 2021-03-01 21:23:40,717] cov:0.2498769584864044\n", + "[INFO 2021-03-01 21:23:40,718] Iteration 7\n", + "[INFO 2021-03-01 21:24:01,997] auc:0.6528805429947431\n", + "[INFO 2021-03-01 21:24:01,997] cov:0.2755841655947051\n", + "[INFO 2021-03-01 21:24:01,999] Iteration 8\n", + "[INFO 2021-03-01 21:24:22,699] auc:0.653538321650494\n", + "[INFO 2021-03-01 21:24:22,700] cov:0.3012489219787817\n", + "[INFO 2021-03-01 21:24:22,701] Iteration 9\n", + "[INFO 2021-03-01 21:24:42,715] auc:0.6544837753109667\n", + "[INFO 2021-03-01 21:24:42,716] cov:0.3182659639579475\n", + "[INFO 2021-03-01 21:24:42,717] Iteration 10\n", + "[INFO 2021-03-01 21:25:01,542] auc:0.6558215983895119\n", + "[INFO 2021-03-01 21:25:01,542] cov:0.3443018600089941\n" + ] + } + ], + "source": [ + "auc_history = {}\n", + "cov_history = {}\n", + "iters = {}\n", + "for strategy in strategies:\n", + " model = CAT.model.IRTModel(**config)\n", + " model.init_model(test_data)\n", + " model.adaptest_load(ckpt_path)\n", + " test_data.reset()\n", + " auc_history[strategy.name] = []\n", + " cov_history[strategy.name] = []\n", + " iters[strategy.name] = []\n", + " \n", + " logging.info('-----------')\n", + " logging.info(f'start adaptive testing with {strategy.name} strategy')\n", + "\n", + " logging.info(f'Iteration 0')\n", + " iters[strategy.name].append(0)\n", + " # evaluate models\n", + " results = model.evaluate(test_data)\n", + " auc_history[strategy.name].append(results['auc'])\n", + " cov_history[strategy.name].append(results['cov'])\n", + " for name, value in results.items():\n", + " logging.info(f'{name}:{value}')\n", + " \n", + " for it in range(1, test_length + 1):\n", + " logging.info(f'Iteration {it}')\n", + " # select question\n", + " selected_questions = strategy.adaptest_select(model, test_data)\n", + " for student, question in selected_questions.items():\n", + " test_data.apply_selection(student, question)\n", + " # update models\n", + " model.adaptest_update(test_data)\n", + " # evaluate models\n", + " results = model.evaluate(test_data)\n", + " # log results\n", + " iters[strategy.name].append(it)\n", + " auc_history[strategy.name].append(results['auc'])\n", + " cov_history[strategy.name].append(results['cov'])\n", + " for name, value in results.items():\n", + " logging.info(f'{name}:{value}')\n", + " writer.add_scalars(name, {strategy.name: value}, it)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABDnklEQVR4nO3dd3iV5fnA8e+dhL0hYa8ACSuBAAEZIiAiCAgqDsQBWEBtFbXVOmqrdVSq/qq2dYEgalVQVECg4GIqAgFZCTMhQMIKAUJCErLu3x/vSXqIENY5ORn357pycc7zrvtNSO7zvM8SVcUYY4zxBD9fB2CMMabssKRijDHGYyypGGOM8RhLKsYYYzzGkooxxhiPsaRijDHGYyypGGOM8RhLKsYUMxFZJiLHRaRSobIJhfbrLyIJbu9FRCaLyFYROSUiCSLyuYiEF2f8xhTFkooxxUhEWgJ9AQVGXOThbwAPAZOBukAoMBcY5rkIjbk8Ab4OwJhy5m7gZ2ANMBb4/EIOEpEQ4HdAL1Vd67bpY49HaMxlsKRiTPG6G/gHTlL5WUQaqOrhCzhuIJBQKKEYU+LY4y9jiomIXAm0AD5T1fVALDDmAg+vBxz0VmzGeIolFWOKz1jgG1U96nr/iasMIAeoUGj/CkC263Uy0MjrERpzmezxlzHFQESqALcC/iJyyFVcCagtIp2BfUDLQocFA3tdr78H3hSRSFWNKoaQjbkkVlMxpnjcAOQCHYAI11d7YCVOO8tsYLyI9HB1HQ4FHgFmAajqLuAt4FNXV+OKIlJZREaLyBPFfTPGnIvYeirGeJ+ILAaiVfUPhcpvBf4JNMVJLn8AmgFHgPeAl1U1z7Wv4HQnnoRTizkOrAKeU9XoYroVY4pkScUYY4zH2OMvY4wxHmNJxRhjjMdYUjHGGOMxllSMMcZ4TLkepxIYGKgtW7b0dRjGGFOqrF+//qiqBp1tW7lOKi1btiQqysaRGWPMxRCRvefaZo+/jDHGeIwlFWOMMR5jScUYY4zHlOs2lbPJzs4mISGBzMxMX4diTLlTuXJlmjZtSoUKhSdsNqWFJZVCEhISqFGjBi1btsSZaskYUxxUleTkZBISEggODvZ1OOYS2eOvQjIzM6lXr54lFGOKmYhQr149e0pQyllSOQtLKMb4hv3ulX6WVIwxphw5mZnNu8tjWbvnmFfO79WkIiJDRGSHiOw+10JCInKriMSISLSIfOJW3lxEvhGRba7tLV3lwSKyxnXO2SJS0VVeyfV+t2t7S2/emzeJCHfeeWfB+5ycHIKCghg+fPglnW/+/PlMmTLFU+FdtP79+9O2bVsiIiKIiIhgzpw5TJgwgZiYmCKP8eTA1JYtW3L06NEi99m+fTsRERF06dKF2NhYj127KBs3bmTRokUF7z35s3rxxRfp2LEjnTp1IiIigjVr1gDw+uuvk56eftHnmzlzJgcOHPBIbKb4JRxP5/kFMfT62/e89N/tLN1xxDsXUlWvfAH+QCzQCqgIbAI6FNonBPgFqON6X99t2zJgkOt1daCq6/VnwGjX63eA+12vfwu843o9Gph9vhi7deumhcXExPyqrLhVq1ZNO3furOnp6aqqumjRIu3cubMOGzbMx5Fdmn79+um6deu8fky+7OzsX5W1aNFCk5KSijzupZde0ueff/6Cr5OXl6e5ubkXHZ+7999/X3/3u99d1jnO5qefftKePXtqZmamqqomJSVpYmKiqhb9vcjJyTnnOS/nZ3IxSsLvYFmyaf9xfeCTDdrqyYXa6smF+tCnG3RLwonLOicQpef4u+rNmkoPYLeqxqlqFs6yqCML7TMReFNVjwOo6hEAEekABKjqt67yNFVNd618dzUwx3X8BzjLtOI69weu13OAgVKKH9AOHTqUhQsXAvDpp59y++23F2xbu3YtvXr1okuXLvTu3ZsdO3YA8Nprr3HPPfcAsGXLFsLCwkhPT2fmzJk88MADAIwbN47777+fnj170qpVK5YtW8Y999xD+/btGTduXME1qlevXvB6zpw5Bdsu9Pjzya+J5ObmMm7cOMLCwggPD+e1114r2Ofzzz+nR48ehIaGsnLlSgByc3N57LHH6N69O506deLdd98FYNmyZfTt25cRI0bQoUOHc143Pj6e9u3bM3HiRDp27Mi1115LRkYGixYt4vXXX+ftt99mwIABAPzjH/8gLCyMsLAwXn/99YLj27Zty913301YWBgrV66kXbt2jBs3jtDQUO644w6+++47+vTpQ0hICGvXrj3nzywrK4u//OUvzJ49m4iICGbPnn3Gzyo+Pp6rr76aTp06MXDgQPbt21fwM5g8eTK9e/emVatWzJkz51f3efDgQQIDA6lUqRIAgYGBNG7cmH/+858cOHCAAQMGFNxn9erV+cMf/kDnzp1ZvXo1zz33HN27dycsLIxJkyahqsyZM4eoqCjuuOMOIiIiyMjIYP369fTr149u3boxePBgDh48CMC6desKakePPfYYYWFhAFx11VVs3LixIMYrr7ySTZs2XfD/GXPh8vKU77cdZvTU1Yz4948s3X6Ee/q0ZMUfB/D66C6ENanlvYufK9tc7hdwM/Ce2/u7gH8X2mcu8DLwI/AzMMRVfgOwAPgSpybzCk7NJxAnUeUf3wzY6nq9FWjqti0WCCwqxvPVVJ6dv1Vvfecnj349O3/reT8FVKtWTTdt2qSjRo3SjIwM7dy5sy5durSgppKSklLwafzbb7/Vm266SVVVc3NztW/fvvrll19qt27ddNWqVap65qfhsWPH6m233aZ5eXk6d+5crVGjhm7evFlzc3O1a9eu+ssvvxTEkO/zzz/XsWPHXtTx7vr166ehoaHauXNn7dy5sx49erTgU29UVJRec801BfseP3684Jjf//73qqq6cOFCHThwoKqqvvvuuwW1iczMTO3WrZvGxcXp0qVLtWrVqhoXF3fW72n+p/M9e/aov79/QZy33HKLfvTRR6qq+swzz+grr7yiqqpRUVEaFhamaWlpmpqaqh06dNANGzbonj17VER09erVqqoF53P/HowfP77g+zNy5Mgif2aFayru74cPH64zZ85UVdXp06cXnGvs2LF68803a25urkZHR2vr1q1/db+pqanauXNnDQkJ0fvvv1+XLVv2q+9FPkBnz55d8D45Obng9Z133qnz588v+Jnk11SysrK0V69eeuTIEVVVnTVrlo4fP15VVTt27Kg//fSTqqo+/vjj2rFjR1VVnTlzpj700EOqqrpjxw492++fqtVULkdGVo5+umavXv3qUm3x+ALt9bfvdOryWE3JyPLodSiipuLrcSoBOI/A+uOs0b1CRMJd5X2BLsA+YDYwDph3uRcUkUk4a3zTvHnzyz2d13Tq1In4+Hg+/fRThg4desa2lJQUxo4dy65duxARsrOzAfDz82PmzJl06tSJe++9lz59+pz13Ndffz0iQnh4OA0aNCA8PByAjh07Eh8fT0RERJGxXcrxH3/8MZGRkb8qb9WqFXFxcTz44IMMGzaMa6+9tmDbTTfdBEC3bt2Ij48H4JtvvmHz5s0Fn85TUlLYtWsXFStWpEePHhc0viE4OLggRvdzu1u1ahU33ngj1apVK4hl5cqVjBgxghYtWtCzZ88zzuf+PRg4cGDB9yf/3Of6mRVl9erVfPnllwDcdddd/PGPfyzYdsMNN+Dn50eHDh04fPjwr46tXr0669evZ+XKlSxdupTbbruNKVOmnLU26e/vz6hRowreL126lJdffpn09HSOHTtGx44duf766884ZseOHWzdupVBgwYBTg2yUaNGnDhxgtTUVHr16gXAmDFjWLBgAQC33HILzz//PK+88gozZsy4qJqtKdqxU1n85+e9fLg6nqNpWXRsXJM3RkcwNLwRFfyLtz+WN5NKIk5NIl9TV5m7BGCNqmYDe0RkJ06SSQA2qmocgIjMBXoCM4DaIhKgqjmFzpl/vQQRCQBqAcmFg1LVqcBUgMjISC3qBp65vuMF36w3jBgxgkcffZRly5aRnPy/W/nzn//MgAED+Oqrr4iPj6d///4F23bt2kX16tWLbFDNfyTi5+dX8Dr/fU5ODnBm187C4wYu5PgLVadOHTZt2sSSJUt45513+Oyzz5gxY8YZ1/H39y84r6ryr3/9i8GDB59xnmXLlhUkgPNxj9nf35+MjIyLirnwdQp/D9y/P/lxF/UzuxTu13Q+OP6av78//fv3p3///oSHh/PBBx+c9Q955cqV8ff3B5yf9W9/+1uioqJo1qwZzz777FnHjagqHTt2ZPXq1WeUnzhx4pwxV61alUGDBjFv3jw+++wz1q9ffwF3aoqy5+gppq+KY876BDKz8xjQNoiJV7WiVyvfjbXzZgpbB4S4emtVxGk8n19on7k4tRREJBAIBeJcx9YWkfz5+q8GYlzVrqU4j9YAxvK/2st813tc23/Qc/22lRL33HMPzzzzTMGn4HwpKSk0adIEcHrkuJdPnjyZFStWkJycfNZn7ReqQYMGbNu2jby8PL766qtLPs/5HD16lLy8PEaNGsULL7zAhg0bitx/8ODBvP322wWf9Hfu3MmpU6c8Hlffvn2ZO3cu6enpnDp1iq+++oq+ffte8vnO9TOrUaMGqampZz2md+/ezJo1C3Bqehdz/R07drBr166C9xs3bqRFixbnvWZ+AgkMDCQtLe2M/0Pux7Vt25akpKSCpJKdnU10dDS1a9emRo0aBT3N8uPPN2HCBCZPnkz37t2pU6fOBd+P+R9VJSr+GJM+jOLq/1vGZ+sSGNm5Cd8+chXvj+9B79aBPh3v47WaiqrmiMgDwBKc9pAZqhotIs/hPI+b79p2rYjEALnAY6qaDCAijwLfuxrb1wPTXKd+HJglIi/gtLdMd5VPBz4Skd3AMZwkVqo1bdqUyZMn/6r8j3/8I2PHjuWFF15g2LBhBeWPPPIIv/vd7wgNDWX69OkMGDCAq6666pKuPWXKFIYPH05QUBCRkZGkpaVd8n0UJTExkfHjx5OXlwfASy+9VOT+EyZMID4+nq5du6KqBAUFMXfuXI/H1bVrV8aNG0ePHj0KrtulS5ezPiq7EOf6mQ0YMIApU6YQERHBk08+ecYx//rXvxg/fjyvvPIKQUFBvP/++xd8vbS0NB588EFOnDhBQEAAbdq0YerUqQBMmjSJIUOG0LhxY5YuXXrGcbVr12bixImEhYXRsGFDunfvXrBt3Lhx3HfffVSpUoXVq1czZ84cJk+eTEpKCjk5OTz88MN07NiR6dOnM3HiRPz8/OjXrx+1av2vUbhbt27UrFmT8ePHX9T3z0BunrIk+hBTV8Sxcf8JaletwAMD2nBXrxbUr1HZ1+EVkFL+Yf6yREZGauGxENu2baN9+/Y+isiY0i8tLa2g9+CUKVM4ePAgb7zxBgAHDhygf//+bN++HT+/sz8osd/BM506ncPnUfuZ/uMe9h/LoEW9qky4MphR3ZpStaJvmsVFZL2q/rqRFJtQ0hjjYQsXLuSll14iJyeHFi1aFDzu+/DDD/nTn/7EP/7xj3MmFPM/R05mMvOneD5es4+UjGy6tajDn4Z2YFCHBvj7ldzRElZTsZqKMSVKef8d3HEolWkr45i3MZGcPGVIx4ZM6NuKbi1KThuU1VSMMaYEU1V+3J3M1JVxrNiZRJUK/tzeozn39AmmZeCF9WosKSypGGOMj2Tl5LFg8wGmrdzDtoMnCaxeiUevDeWOK1pQp1pFX4d3SSypGGNMMcvMzuWTNfuYuiKOQyczCalfnZdHdWJERGMqV/D3dXiXxZKKMcYUk8zsXD5es493lseSlHqaK4Lr8tKocPqFBOFXghvfL4Z1wSiB3CdzXLRoEaGhoezdu5dnn32WV199tchjbYp3m+LdlDwZWbm8tzKOvi8v5fkFMbQJqs6sST2ZfW8vBrStX2YSClhNpUT7/vvvmTx5MkuWLCkYDe0Jc+fO5eabb+bpp5++oP3zJ4q7nG6gGzduJCoqqmAesxEjRjBixIhLPl++1atXs2DBAjZs2EClSpU4evQoWVlZgJNU7rzzTqpWrfqr43JzcwumJils5syZhIWF0bhx48uOz5RvGVm5fLxmL+8sj+No2ml6t67Hv2/vwhWt6vk6NK+xmkoJtWLFCiZOnMiCBQto3br1RR9vU7zbFO/Gd9Kzcpi2Io6+L//ACwu30bZhdT67txefTOxZphMK4L2p70vD13kX6Vr0uOqMoZ79WvT4r65ZWEBAgNapU0c3bdp0Rrn71OznYlO8l54p3s3Zleap70+dztapy2O12/PfaIvHF+iYaat1TVzy+Q8sZfDRIl3mElWoUIHevXszffr08+9chIud4r169eoFU7wD55zi3c/Pr8gp3m+55RbCwsJ45JFHiI6OPm+cq1evZsyYMYAzxfuqVasKtl3oFO9Tp04lKCiI22677YwJG92dbYr3K664gvDwcH744Yezxuo+xXtERAQvvPACCQkJZ53iPd8tt9zCggULyM7Otiney4n0rBymroil79+X8uKibbRrWJPP7+vFxxN60iO4rq/DK1bWplKU63yzrrufnx+fffYZAwcO5G9/+xtPPfXUJZ3Hpng/k03xbjwtPSuHj1bvZeqKOJJPZdE3JJCHBoYQ2bJ8JRJ3VlMpoapWrcrChQv5+OOPL7vGUhSb4t2meDcX79TpHN5ZHsuVf1/KS//dTofGNfni/l589JsrynVCAauplGh169Zl8eLFXHXVVQQFOUvLvPDCCwWN6QAJCQmXdQ2b4t2meDcX7tTpHD5cvZdpK+M4diqLq0KDeGhgSImal8vXbEJJm1DSXKLLneLdnF1J/B1MO53Dh6vjmbYijuPp2fQLDeKha0Lo2rx8JhObUNIYL7Ap3su+tNM5fPBTPO+tdJJJ/7ZOzaRLOU0mF8JqKlZTMaZEKQm/g/nJZNrKOE6kZzOgbRAPXRNKRLPaPo2rpLCaykVSVZ+u8WxMeeXrD7mpmdkFbSYn0rO5ul19Jg8MsWRyESypFFK5cmWSk5OpV6+eJRZjipGqkpycTOXKxb/eempmtqtmsoeUDCeZPDQwhM6WTC6aJZVCmjZtSkJCAklJSb4OxZhyp3LlyjRt2rTYrheXlMZXvyTy4eq9pGRkM7BdfR66JoROTWsXWwxljSWVQipUqEBwcLCvwzDGeMmRk5nM33SA+ZsOsDkhBRGcZDIwlPCmtc5/AlMkryYVERkCvAH4A++p6q+GqIvIrcCzgAKbVHWMqzwX2OLabZ+qjnCVrwRquMrrA2tV9QYR6Q/MA/a4tn2pqs954baMMaXMycxsFm89xLyNiayOTSZPIbxJLZ4e1p7rOzemQc3if+RWVnktqYiIP/AmMAhIANaJyHxVjXHbJwR4EuijqsdFpL7bKTJUNaLweVW1r9vxX+AkknwrVXW4Z+/EGFMaZWbnsmzHEeb+coAfdhwhKyePFvWq8sDVIYzo3Jg29auf/yTmonmzptID2K2qcQAiMgsYCcS47TMReFNVjwOo6pELPbmI1ASuBmzIsjEGgNw85ee4ZOZtTOS/Ww+RmplDYPWKjOnRnBu6NKFz01rWAcfLvJlUmgD73d4nAFcU2icUQER+xHlE9qyqLnZtqywiUUAOMEVV5xY69gbge1U96VbWS0Q2AQeAR1X1V9POisgkYBJA8+bNL+G2jDEliaqyNfEk8zYm8vXmAxw+eZrqlQIY3LEhIyMa07t1PQL8bRBqcfF1Q30AEAL0B5oCK0QkXFVPAC1UNVFEWgE/iMgWVXVf+/Z24D239xtcx6SJyFBgruvcZ1DVqcBUcAY/ev6WjDHFIf7oKeZtPMC8TYnEJZ2igr/Qv219bohowsD29alc4ewrexrv8mZSSQSaub1v6ipzlwCsUdVsYI+I7MRJBOtUNRFAVeNEZBnQBYgFEJFAnMdrN+afyL3GoqqLROQtEQlU1aIXbDfGlBpHUjNZsOkg8zYdYNP+E4jAFcF1mdi3FUPDGlGragVfh1jueTOprANCRCQYJ5mMBsYU2mcuTo3jfVeiCAXiRKQOkK6qp13lfYCX3Y67GVigqgULYIhIQ+CwqqqI9MCZ1j/ZO7dmjCkuqZnZLIk+zLyNify4+yh5Ch0a1eSpoe0Y3qkxjWtX8XWIxo3Xkoqq5ojIA8ASnPaSGaoaLSLP4SxFOd+17VoRiQFygcdUNVlEegPvikgeTnKY4t5rDCdBFe6efDNwv4jkABnAaPX1nA/GmEtyOieXZTuSmL/xAN9tO8zpnDya1a3Cb/u3YWREY0Ia1Dj/SYxP2ISShSaUNMb4Rl6esmbPMeZtTGTRloOczMyhbrWKDO/UiJERTejavLb13CohbEJJY0yJdSI9i4/X7OM/P+/lYEomVSv6M7hjQ0ZENObKNoFUsJ5bpYolFWOMT+xNPsWMVXv4LCqBjOxc+oYE8uTQ9lzTvj5VK9qfptLKfnLGmGK1Yd9xpq2IY0n0Ifz9hBGdmzChbzDtG9X0dWjGAyypGGO8LjdP+TbmMNNWxrF+73FqVg7g3n6tGde7pc27VcZYUjHGeE1GVi5z1u9n+qo9xCen07ROFZ65vgO3RjajWiX781MW2U/VGONxSamn+XB1PB/9vJcT6dl0blabNwe3Y3DHBjZlShlnScUY4zG7Dqfy3so9fPVLItl5eVzTvgGTrmpFZIs61h24nLCkYoy5LKrK6rhkpq2IY+mOJCoF+HFLZFN+c2UwrYJsevnyxpKKMeaSZOfmsWjLQaauiCP6wEnqVavII9eEcmfP5tSrXsnX4RkfsaRijLkoJzOzmb12P+//uIcDKZm0DqrGSzeFc2OXJjYzsLGkYoy5MAdOZPD+j3v4dO1+0k7n0LNVXZ6/IYwBbevj52ftJcZhScUYU6StiSlMWxnHgs0HARgW3oiJfVsR3rSWjyMzJZElFWPMr+TlKct3JjF1RRyr45KpXimA8b1bMq5PS5rWqerr8MzlSjsC/hWgSh2Pn9qSijGmQGZ2LvM2JjJt5R52H0mjYc3KPDW0HaN7NKdmZVsAq9RLPwY//RPWvAuR98DgFz1+CUsqxpRzp3NyWbnzKIu2HOTbmMOkns6hQ6OavH5bBMM6NbJZgsuCzJPw89uw+t9wOhXCb4Zu471yKUsqxpRD+Ylk4ZaDfOdKJLWqVGBIWENu7NKEXq3r2WDFsiDrFKydBj++DhnHod1wGPAnaNDBa5e0pGJMOZGZncvKXU6NxD2RXBfekKHhjejdOpCKAVYrKRNyTsP6mbDiVTh1BNoMgqv/BI27eP3SllSMKcPcE8m3MYdJK5RI+tgiWGVLbjZs/BiWvwInE6BlX7jtI2jes9hCsKRiTBmTn0gWbj7Ad9uOFCSSoeENGdapMb1b17NEUtbk5cKWz2HZS3A8Hpp2hxvehOB+UMyPMS2pGFMGZGbnsmJnkvNoy5VIaletwLDwRgzt1MgSSVmVlwfb5sPSv8HRHdAwHMZ8BiHXFnsyyefVpCIiQ4A3AH/gPVWdcpZ9bgWeBRTYpKpjXOW5wBbXbvtUdYSrfCbQD0hxbRunqhvFaVV8AxgKpLvKN3jp1ozxOUsk5Zgq7FwCS1+AQ1sgsC3c8gG0HwF+vv2Zey2piIg/8CYwCEgA1onIfFWNcdsnBHgS6KOqx0WkvtspMlQ14hynf0xV5xQquw4IcX1dAbzt+teYMiM/kSzccpDvLZGUP6oQtwx+eAESo6BOMNw41eki7Fcy5l3zZk2lB7BbVeMARGQWMBKIcdtnIvCmqh4HUNUjl3G9kcCHqqrAzyJSW0QaqerByzinMT6XmZ3LcleNpHAiGdapEb0skZQPe1c7yWTvKqjZBK5/AyLucEbGlyDeTCpNgP1u7xP4dc0hFEBEfsR5RPasqi52bassIlFADjBFVee6HfeiiPwF+B54QlVPn+N6TYAzkoqITAImATRv3vySb84YbzpXIhneqRFDwy2RlCuJG2Dpi7D7O6hWH657GbqOhQqVfR3ZWfm6oT4A53FVf6ApsEJEwlX1BNBCVRNFpBXwg4hsUdVYnMdlh4CKwFTgceC5C72gqk51HUdkZKR68F6MuWzZuXl88FM8//x+Fyczc6hjiaT8OhztNMBvX+DM0TXoOeg+ESqW7LnXvJlUEoFmbu+busrcJQBrVDUb2CMiO3GSzDpVTQRQ1TgRWQZ0AWLdHmedFpH3gUcv4nrGlFgrdibx16+jiU06Rf+2QdzTJ9gSSXl0dLfTNXjrF1CpBvR/CnreD5Vr+jqyC+LNpLIOCBGRYJw/7qOBMYX2mQvcDrwvIoE4j8PiRKQOkK6qp13lfYCXAfLbSVy9vW4AtrrONR94wNV2cwWQYu0ppjTYm3yK5xds47tthwkOrMaMcZFc3a6Br8Myxe34Xlj+Mmz6BAIqw5WPQO8HoWpdX0d2UbyWVFQ1R0QeAJbgtJfMUNVoEXkOiFLV+a5t14pIDJCL06srWUR6A++KSB7gh9Omkt/A/7GIBAECbATuc5UvwulOvBunS7F3ZkszxkNOnc7hrWW7mbZiDxX8hSeua8f4Pi2pFFAyevGYYnLygDOdyoYPQfzgivuchFK9/vmPLYHE6SxVPkVGRmpUVJSvwzDljKoyf9MB/rZoG4dPnuamrk14Ykg76tcsmQ2vxkuO74W1U2Hde5CXA13vhr6PQq0mvo7svERkvapGnm2brxvqjSlXtiam8Oz8aKL2Hie8SS3euqMb3Vp4fqEkU0JlpkDMPNg0C/b+6NRMOo2G/o9DnZa+js4jLKkYUwyS007z6jc7mbVuH3WrVuTlUZ24uVtTW9u9PMjNhtgfYNOnsOO/kJMJ9drAgKeh061Qp4WvI/QoSyrGeFF2bh7/+Xkvr327k/SsXO7pE8zkgSHUqlKyBqwZD1OFgxth02xnosf0o1ClLnS5CzqPhibdfDY3l7dZUjHGS37cfZS/fh3NzsNp9A0J5JnrO9Cmfg1fh2W8KSUBNn8Gm2dD0nbwrwihQ5xE0mYQBFT0dYReZ0nFGA/bfyydFxduY3H0IZrVrcLUu7oxqEMDW0mxrDqdCjHzYfMs2LMSUGjWE4a/Bh1vdAYuliOWVIzxkIysXN5eHsu7y2PxE+GxwW35zZXBVK5gXYTLnNwcZ2LHzbNg2wLIyXAmd+z/hNNOUreVryP0GUsqxlwmVWXhloP8beE2DqRkMqJzY54c2o5Gtar4OjTjaYe2OD23tnwOaYehci2IuN3pwdWsR5ltJ7kYllSMuQzbDp7k2fnRrNlzjA6NavL66C70CC5dI6DNeZw86CSRTbPgSDT4VXAWweo8GkIHQ0AlX0dYolhSMeYSHD+VxT++3cnHa/ZSq0oFXrwxjNHdm+NvXYTLhqxTzmOtzbOcx1yaB00iYeirEDaq1E2dUpwsqRhzEXJy8/h07T7+79udpGbmcHevljx8TQi1q5b9Xj1lXl4u7Fnh9NyKmQ/Zp6B2c+j7B+fxVmAbX0dYKlhSMeYC/RyXzLPzo9l+KJVererxzIgOtGtYOmaONeeQlweHNkP0l7D5c0g9AJVqQvgo6Hy704vLx8vzljaWVIw5jwMnMvjbom0s2HyQJrWr8PYdXRkS1tC6CJdGeXmQtM3p+hu/EuJXQeYJEH8IGQSDX4S210EF62RxqSypGHMOmdm5TF0Rx1vLdqMKD18Twr1XtaZKResiXGqowtGdzmOt/CSSnuxsq90C2g2H4L7QeiBUD/JtrGWEJRVjCknJyOY/P+/l/R/3cDQti2HhjXhyaDua1inZK+4ZnCRyLO7MJJJ22NlWs4kzqj24L7TsW+bm3CopLKkY43IoJZPpq+L4ZM0+TmXl0i80iN8NaGNdhEu64/H/e5y1Z6XTLgJQvYGTPPKTSN1WNo6kGFhSMeXe7iNpTF0Ry1e/JJKnMLxTI+69qjUdGlsjfImUknBmEknZ55RXDYSWV7qSyFUQGGJJxAcsqZhya8O+47yzLJZvtx2mUoAfY3o0Z0LfVjSra4+5SpTUQ64kssL59/gep7xKHWjRB3o/4NRE6re3JFICWFIx5YqqsmxnEu8si2XNnmPUqlKBBwe0YWzvltSrbiOjS4S0JFd7iKsmkrzLKa9UC1r0hh4TnSTSIMy6+5ZAllRMuZCTm8fCLQd5e1ks2w+l0qhWZZ4e1p7bezSnWiX7NfCprFNOg/ru750G9qRtTnnF6tC8F3S9y0kijTqDn/W8K+nst8mUaRlZuXwWtZ9pK+NIOJ5BSP3qvHpLZ0Z0bkzFAPuU6xOqcHQX7P4Wdn0Le3+C3NMQUAWa94ROtzhtIo0jwN8WMyttLKmYMulEehYfrt7LzJ/iOXYqi24t6vDM9R0Z2K6+LeHrC1mnnFrIrm+dZHLC1bgeGArdJ0DINdC8N1So7Ns4zWXzalIRkSHAG4A/8J6qTjnLPrcCzwIKbFLVMa7yXGCLa7d9qjrCVf4xEAlkA2uBe1U1W0T6A/MAVyseX6rqc965M1NSHTiRwXsr9zBr3T7Ss3IZ2K4+9/VvTfeW1i24WKlC0g4ngez+zlUbyYIK1aBVP+jzMLS5xsaKlEFeSyoi4g+8CQwCEoB1IjJfVWPc9gkBngT6qOpxEanvdooMVY04y6k/Bu50vf4EmAC87Xq/UlWHe/ZOTGmw63Aq7yyPY97GRABGdG7Mvf1a07ahLd9bbE6nwZ7lrtrI9//r6hvUDnpMcqZBad7Lpoov486ZVERkMFBDVecUKr8ZSFHVb89z7h7AblWNcx03CxgJxLjtMxF4U1WPA6jqkfMFrKqL3GJZCzQ93zGm7IqKP8Y7y2P5btsRqlTw565eLZjQtxVNatvcTV6n6qzDnv9Ia+9qyMt2Gthb9Ye+jzi1kdrNfR2pKUZF1VT+AtxwlvJlwNfA+ZJKE2C/2/sE4IpC+4QCiMiPOI/InlXVxa5tlUUkCsgBpqjqXPcDRaQCcBfwkFtxLxHZBBwAHlXV6MJBicgkYBJA8+b2n700ystTlu44wjvLY1kXf5w6VSvw8DUhjO3VkjrVbAp6r8o8eWZt5GSCU16/A/S836mNNOsJAfZzKK+KSiqVVDWpcKGqHhWRah68fgjQH6fGsUJEwlX1BNBCVRNFpBXwg4hsUdVYt2PfAlao6krX+w2uY9JEZCgw13XuwvFPBaYCREZGqofuwxSD7Nw85m88wLsrYtl5OI0mtavw7PUduLV7M6pWtD4nXqEKR2JcSeQ72Lca8nKgYg2nbaTfY05tpJY9MDCOon4Ta4pIgKrmuBe6aggX8mwhEWjm9r6pq8xdArBGVbOBPSKyEycRrFPVRABVjRORZUAXINYVwzNAEHBv/olU9aTb60Ui8paIBKrq0QuI1ZRg6Vk5zFq7n+mr9pB4IoO2DWrw2m2dGd6pMRX8rVuwx2WmOKsd5tdG8ufSahAGvR5waiNNe1htxJxVUUnlS2CaiDygqqcARKQ6Tm+uLy/g3OuAEBEJxkkmo4ExhfaZC9wOvC8igTiPw+JEpA6QrqqnXeV9gJddMUwABgMDVTUv/0Qi0hA4rKoqIj0APyD5AuI0JZSqMn/TAZ77OobkU1n0aFmXF24Io3/bIFvLxBtOHoAlT8G2r53aSKWaTttIyCCnNlKzsa8jNKVAUUnlaeAFYK+I7AUEp+YxHfjz+U6sqjki8gCwBKe9ZIaqRovIc0CUqs53bbtWRGKAXOAxVU0Wkd7AuyKSh5Mcprj1GnsH2Ausdv1hye86fDNwv4jkABnAaFW1x1ulVOKJDJ7+agtLdyQR0aw2U+/uRrcW1i3YK/JyYd10+P45p6H9ivug7VBo1sMGH5qLJuf7uysiVYD8xZl3q2qG16MqJpGRkRoVFeXrMIyb3Dzlo9XxvLxkBwCPXtuWsb1b4m8DFr3j4Gb4+iE4sAFaXw3D/s+ZIt6YIojIelWNPNu2oroU31SoSIHaIrJRVVM9GaAxADsPp/L4F5v5Zd8JrgoN4sUbwmzGYG/JOgVL/wY/vw1V68Ko6RA2ymb5NZetqMdf15+lrC7QSUR+o6o/eCkmU86czsnlraWxvLVsN9UrBfD6bRGMjGhs7SbesnMJLPwDpOyHrmNh0F+daeSN8YBzJhVVHX+2chFpAXzGr8ecGHPR1u89zhNfbGbXkTRuiGjMn4d3sCnoveXkQVj8OMTMc0a5j18MLXr5OipTxlx0535V3evqVmzMJUs7ncMri7fz4c97aVSzMu+P686AdvXPf6C5eHm5EDXDaYjPOQ1X/xl6T7YuwcYrLjqpiEg74LQXYjHlxA/bD/P0V1s5eDKTsb1a8ujgtlS3NU2849BWpyE+McrpHjzsH1Cvta+jMmVYUQ31X+M0zrurCzTifxM6GnPBktNO89evY5i/6QAh9asz577edGthz/K9IusULP87/PRvp73kpmkQfos1xBuvK+rj4auF3itwDCex3Ams9lZQpmxRVb76JZHnF8SQdjqHh68J4bf929giWd6y61tY+HtnzZIud8Gg55weXsYUg6Ia6pfnvxaRLjij4W/BWa/kC++HZsqC/cfSeeqrLazcdZSuzWvz91GdCGlg09F7ReohWPwkRH/pLH41bhG07OPrqEw5U9Tjr1CcKVRuB44Cs3EGSw4opthMKZabp8z8KZ5Xl+zAT+C5kR2584oWtuqiN+Tlwfr34bu/Qk4mDPgT9HnI1i0xPlHU46/twEpguKruBhCRR4olKlOqbT90kse/2MKm/ScY0DaIF24Mt/VNvOVwjNMQn7AWgq+CYa9BYJvzH2eMlxSVVG7CmQRyqYgsBmbhzP9lzFllZufy5tLdvL0sllpVKvDP27twfadGNojRG7LSYcXL8NO/nIkfb3gHOo+2hnjjc0W1qcwF5rrWThkJPAzUF5G3ga9U9ZtiidCUCuvij/HEF5uJTTrFTV2b8OdhHWzBLG/Z/Z0zIv54PETc6TTEV6vn66iMAS5gnIpr2vtPgE9cU9LfAjwOWFIxpGZm8/fF2/nPz/toUrsKH9zTg36hQb4Oq2xKO+I0xG+dA/VCYOwCCO7r66iMOcNFjThzrSVfsHKiKd++iznM03O3ciQ1k99cGczvB4VSzQYxel5eHvzyIXz7F8jOgP5PwpWPWEO8KZHsL4C5aEmpp3n262gWbj5Iu4Y1eOeubkQ0q+3rsMqmI9thwcPOMr4t+8Lw1yDwV6tkG1NiWFIxF0xVmbM+gRcWbiMjK5dHrw1l0lWtbRCjN2RnwIpX4cc3oFJ1GPkWRIyxhnhT4llSMRdkx6FUnl8Qw6rdR+nesg4v3dSJNvWr+zqssiclATZ/Butnwom90HkMXPs8VAv0dWTGXBBLKuacVJV18cd5Z3ksP2w/QvVKAbxwQxhjejS3QYyelJkCMfNh82yIXwUoNLsCRvwLWvXzdXTGXBRLKuZX8vKU77Yd5p3lsWzYd4K61Sry+0Gh3NWzhXUT9pTcbKdr8ObZsOO/zkj4uq1hwFPOxI91g30doTGXxJKKKZCVk8fcXxJ5d0UssUmnaFqnCs+N7Mgt3ZpRpaK/r8Mr/VQhIcpJJFu/gIxjULUedL0bOo2GJl2tzcSUel5NKiIyBHgD8AfeU9UpZ9nnVuBZnFmQN6nqGFd5LrDFtds+VR3hKg/GGd1fD1gP3KWqWSJSCfgQ6AYkA7eparz37q7sSM3M5tO1+5i+ag+HT56mfaOavDE6gmHhjQjwt0b4y3Yszmkn2TzbeR1QGdoOhU63QZuB4G9r3pmyw2tJRUT8gTeBQUACsE5E5qtqjNs+IcCTQB9VPS4i7kv/ZahqxFlO/XfgNVWdJSLvAL8B3nb9e1xV24jIaNd+t3nj3sqKpNTTvP/jHj76eS+pmTn0bl2PV27uTN+QQJta5XKlH3NmC94025mXC4GWV0LfP0D766FyLV9HaIxXeLOm0gPYrapxACIyC2e6lxi3fSYCb7oGVaKqR4o6oTh/6a7GmYYf4AOcWs7brnM/6yqfA/xbRERVCy80Vu7FHz3F1JVxzFmfQHZuHteFNeTeq1rT2caaXJ7sTNi52KmR7PoW8rIhqD1c86zTTlKrqa8jNMbrvJlUmgD73d4nAFcU2icUQER+xHlE9qyqLnZtqywiUUAOMMU1F1k94ISq5rids0nh66lqjoikuPY/6n5BEZkETAJo3rz5Zd5i6bI54QTvLI/lv1sPUcHfj1FdmzLpqlYEB1bzdWilV16eMzBx8yyIngenU6B6Q7jiXmeCxwZh1k5iyhVfN9QHACFAf6ApsEJEwlX1BNBCVRNFpBXwg4hsAVIu94KqWjDNTGRkZJmvxagqK3cd5Z3lsfwUm0yNygHc16814/u0pH6Nyr4Or/RK2gGbZsGWzyFlP1SoBh1GQKdbIbgf+FnHBlM+eTOpJALN3N43dZW5SwDWqGo2sEdEduIkmXWqmgigqnEisgzogrPiZG0RCXDVVtzPmX+9BBEJAGrhNNiXSzm5eSzaeoh3l8cSfeAkDWpW4qmh7bi9R3NqVLaG4UuSetjptbV5FhzcBOIHra+Ggc9Au6FQ0Wp8xngzqawDQly9tRJx1mYZU2ifuTgrS74vIoE4j8PiXLMhp6vqaVd5H+BlVVURWQrcjNMDbCwwz3Wu+a73q13bfyiP7SmZ2bl8HrWfqSvj2H8sg1ZB1Xh5VCdGdmlMpQD79HzRsk7B9oVOO0nsD6B50CgCBr8EYaOgRgNfR2hMieK1pOJq13gAWILTXjJDVaNF5DkgSlXnu7ZdKyIxQC7wmKomi0hv4F0RyQP8cNpU8hv4HwdmicgLwC/AdFf5dOAjEdkNHMNJYuXGifQsPly9lw9+iif5VBZdmtfm6WEdGNS+gY1+vxTZGbBsCqydBtmnoFYzZ2bgTrdBUFtfR2dMiSXl8MN8gcjISI2KivJ1GJflwIkM3lu5h1nr9pGelcuAtkHc1681PYLrWrfgS7VnJXw92RlTEn4rdBsHzXuBn43ZMQZARNarauTZtvm6od5cop2HU3lneSzzNx4AYETnxkzq14p2DWv6OLJSLDPFWbNk/UyoEwxjv3bWfTfGXDBLKqVIbp6yJi6Z6av28P32I1Sp4M9dvVowoW8rmtSu4uvwSrfti2Dh7yHtMPR+EPo/BRWr+joqY0odSyolXFZOHj/FHmVJ9CG+iT5M8qks6laryCPXhHJ3L5vg8bKlJcF//+iMfq/fEUZ/DE26+ToqY0otSyolUEZWLst3JrEk+hDfbTtMamYO1Sr6c3X7Bgzp2JCr29W3CR4vl6rTo2vxE04PrwFPQ5+HIMCStDGXw5JKCZGamc0P24+weOshlu1IIiM7l9pVKzCkY0OGhDWkT5tAKlewROIRJ/bB1w9D7Pf/W7fEenQZ4xGWVHzo2Kksvos5zOLoQ6zadZSs3DyCalRiVLcmXBfWiB7BdalgswR7Tl4erJsG3/3VeX/dK9B9gvXqMsaDLKkUs8MnM1kSfYjFWw+xZs8xcvOUJrWrcHevFgwJa0jX5nVsXIk3JO2A+Q/C/jXQ5hoY/hrULl9zvxlTHCypFIN9yeksjj7I4q2H2LDvBACtg6pxf7/WDAlrSMfGNW1MibfkZMGPb8CKl51pVG581xnAaN9vY7zCkoqX7DqcyuKth/jv1kPEHDwJQMfGNfnDoFCuC29Im/o1fBxhOZC4HuY9CEeioeNNcN3LUD3I11EZU6ZZUvEQVWVr4smCGkls0ikAurWow5+Gtmdwx4Y0r2fjHopFVjosfRF+fguqN4DRnzoTPhpjvM6SymXIy1M27DvOf7c6bSSJJzLw9xOuCK7LuN4tubZjQxrUtOnli1XccmeKlePx0G08DPqrrbJoTDGypHIJdhxK5aOf41kSfZik1NNU9PfjypBAHhoYwjUdGlDXBiQWv4wT8O2fYcOHULcVjF0AwX19HZUx5Y4llUtw4EQGX6xPZEC7IAa7BiPaGiU+tG0BLPwDnDriDGDs/yRUsGlrjPEFSyqX4MqQQDb8eZCNave1tCOw6DGImQsNwmHMLGjcxddRGVOuWVK5BBX8/bDB7T6kCps+hcVPOuueXP1np4bib7VFY3zNkoopXY7vhQUPO6swNuvpmmIl1NdRGWNcLKmY0iEvF9ZOhe+fdwYuDn0VIn9jU6wYU8JYUjEl35HtMP8BSFgHbQa5plhp5uuojDFnYUnFlFyHo2HVa7D1S2esyU3TIPwWm2LFmBLMkoopefb9DCv/AbuWQMXq0PN+uPIRqBbo68iMMefh1QfSIjJERHaIyG4ReeIc+9wqIjEiEi0inxTaVlNEEkTk3673NURko9vXURF53bVtnIgkuW2b4M17Mx6mCju/gRlDYMZgSIxyFs56eAsMftESijGlhNdqKiLiD7wJDAISgHUiMl9VY9z2CQGeBPqo6nERqV/oNM8DK/LfqGoqEOF2/HrgS7f9Z6vqA56+F+NFuTnOOJNVr8HhrVCrmTPxY5c7nVmFjTGlijcff/UAdqtqHICIzAJGAjFu+0wE3lTV4wCqeiR/g4h0AxoAi4HIwicXkVCgPrDSWzdgvCg7EzZ+DD/905mnK7At3PC202Zi402MKbW8mVSaAPvd3icAVxTaJxRARH4E/IFnVXWxiPgB/wfcCVxzjvOPxqmZqFvZKBG5CtgJPKKq+wsfJCKTgEkAzZvbIk3FLvMkRE2H1W8506o06QbXvghth1r3YGPKAF831AcAIUB/oCmwQkTCcZLJIlVNKGLxqtHAXW7vvwY+VdXTInIv8AFwdeGDVHUqMBUgMjJSC283XpJ2BH5+G9ZNh9Mp0Ppqp/G9ZV/rzWVMGeLNpJIIuA8maOoqc5cArFHVbGCPiOzESTK9gL4i8lugOlBRRNJU9QkAEekMBKjq+vwTqWqy23nfA1729A2ZS3A8Hn76F/zyH8g5DR1GwpUP2xxdxpRR3kwq64AQEQnGSSajgTGF9pkL3A68LyKBOI/D4lT1jvwdRGQcEJmfUFxuBz51P5GINFLVg663I4BtnrsVc9EOR8Oq12HrFyB+EHE79H4IAtv4OjJjjBd5Lamoao6IPAAswWkvmaGq0SLyHBClqvNd264VkRggF3isUI3jXG4FCi/lN1lERgA5wDFgnIduxVyMfWtg1T9g52KoUM0ZY9Lrd1Czsa8jM8YUAzmznbt8iYyM1KioKF+HUfqpwu7vnAGL+36CKnWdZNJ9AlSt6+vojDEeJiLrVfVXvXLB9w31pjQrGGPyOhzeAjWbwpC/Q9e7bIyJMeWUJRVz8bIzYdMn8OMbZ44xCbsZAmwpZWPKM0sq5sJlnoSoGfDzW5B22MaYGGN+xZKKuTC//AcWP+WMMWk1wJkxOPgqG2NijDmDJRVTtLxc+PYvsPrfzkDFQc9Bk66+jsoYU0JZUjHnlpkCc34Du7+FK+5zHnX5238ZY8y52V8Ic3bH4uCT0XAsFoa/DpHjfR2RMaYUsKRifm3PSvjMNa3aXV85bSfGGHMBrMuOOVPU+/DRDVCtPkz43hKKMeaiWE3FOHJzYMlTsPZdaDMIbp7urAtvjDEXwZKKgYwTMGc8xP4AvR5wenj5+fs6KmNMKWRJpbxLjoVPbnNGxo/4F3S929cRGWNKMUsq5VnsUvh8nFMruXsetOzj64iMMaWcNdSXV2unwX9GOVPST/zBEooxxiOsplLe5GbDfx931okPvQ5GTYNKNXwdlTGmjLCkUp6kH3Med+1ZDn0egoHPWIO8McajLKmUF0k74dPbICXBmaY+ovDKzsYYc/ksqZQHu7+Dz+9x1joZuwCaX+HriIwxZZQ11JdlqvDz2/DxLVC7udMgbwnFGONFVlMpq3KyYNGjsOEDaDccbnwXKlX3dVTGmDLOkkpZlH4MZt8Fe1dB3z/AgKdtZUZjTLHw6l8aERkiIjtEZLeIPHGOfW4VkRgRiRaRTwptqykiCSLyb7eyZa5zbnR91XeVVxKR2a5rrRGRlt68txLryHaYNgAS1jmrMw78iyUUY0yx8VpNRUT8gTeBQUACsE5E5qtqjNs+IcCTQB9VPZ6fINw8D6w4y+nvUNWoQmW/AY6rahsRGQ38HbjNQ7dTOuz8BubcAxWrwvhF0DTS1xEZY8oZb36E7QHsVtU4Vc0CZgEjC+0zEXhTVY8DqOqR/A0i0g1oAHxzgdcbCXzgej0HGChSThZQV4Wf/gWf3Ar1WsHEpZZQjDE+4c2k0gTY7/Y+wVXmLhQIFZEfReRnERkCICJ+wP8Bj57j3O+7Hn392S1xFFxPVXOAFKBe4QNFZJKIRIlIVFJS0qXeW8mRcxrmPQDfPA0dRsD4/0Ktwt9mY4wpHr5+2B4AhAD9gduBaSJSG/gtsEhVE85yzB2qGg70dX3ddTEXVNWpqhqpqpFBQUGXE7vvnToKH46Ejf+Bfo/DzTOhYjVfR2WMKce82fsrEWjm9r6pq8xdArBGVbOBPSKyEyfJ9AL6ishvgepARRFJU9UnVDURQFVTXQ37PYAP3a6XICIBQC0g2Xu352OHo5015E8dgZtnQNgoX0dkjDFeramsA0JEJFhEKgKjgfmF9pmLU0tBRAJxHofFqeodqtpcVVviPAL7UFWfEJEA136ISAVgOLDVda75wFjX65uBH1RVvXVzPrV9EUy/FvKyncddllCMMSWE12oqqpojIg8ASwB/YIaqRovIc0CUqs53bbtWRGKAXOAxVS2qdlEJWOJKKP7Ad8A017bpwEcishs4hpPEypaM484I+eUvQ+MuMPoTqNnI11EZY0wBKasf5i9EZGSkRkUV7plcAh3YCOumwZYvICcDwm9xVmmsUMXXkRljyiERWa+qZ+1iaiPqS6rsTIiZC+vecwYyVqgKnW+D7hOgYbivozPGmLOypFLSnNgHUTNgw4eQngz12sCQKdD5dqhS29fRGWNMkSyplAR5eRC31KmV7FzslLUd6tRKgvvZNCvGmFLDkoovZRyHjZ/AuulwLBaqBsKVj0C38VC72fmPN8aYEsaSii8c3ARrp8GWOU7De7MroP+Tzoj4gEq+js4YYy6ZJZXiknMaYuY5ySRhLQRUgU63QPeJ0KiTr6MzxhiPsKTibSf2uzW8H4W6rWHwS84a8dbwbowpYyypeENBw/t02Plfpyx0iNPw3mqANbwbY8osSyqelHHC1fD+3v8a3vs8DJHjnTXijTGmjLOk4gkHNzsj3jd/7jS8N+3hzBrc8QZreDfGlCuWVC5VzmmIme8kk/1rnIb38JudR1yNI3wdnTHG+IQllUuxcwnM+x2cSoK6reDaF6HLHVCljq8jM8YYn7KkcinqBEOTSOgxAVpdbQ3vxhjjYknlUgSFwphZvo7CGGNKHPuIbYwxxmMsqRhjjPEYSyrGGGM8xpKKMcYYj7GkYowxxmMsqRhjjPEYSyrGGGM8xpKKMcYYjxFV9XUMPiMiScDeSzw8EDjqwXBKA7vn8sHuuXy4nHtuoapBZ9tQrpPK5RCRKFWN9HUcxcnuuXywey4fvHXP9vjLGGOMx1hSMcYY4zGWVC7dVF8H4AN2z+WD3XP54JV7tjYVY4wxHmM1FWOMMR5jScUYY4zHWFK5BCIyRER2iMhuEXnC1/F4m4g0E5GlIhIjItEi8pCvYyoOIuIvIr+IyAJfx1JcRKS2iMwRke0isk1Eevk6Jm8SkUdc/6e3isinIlLZ1zF5g4jMEJEjIrLVrayuiHwrIrtc/3pkPXRLKhdJRPyBN4HrgA7A7SLSwbdReV0O8AdV7QD0BH5XDu4Z4CFgm6+DKGZvAItVtR3QmTJ8/yLSBJgMRKpqGOAPjPZtVF4zExhSqOwJ4HtVDQG+d72/bJZULl4PYLeqxqlqFjALGOnjmLxKVQ+q6gbX61ScPzRNfBuVd4lIU2AY8J6vYykuIlILuAqYDqCqWap6wqdBeV8AUEVEAoCqwAEfx+MVqroCOFaoeCTwgev1B8ANnriWJZWL1wTY7/Y+gTL+B9adiLQEugBrfByKt70O/BHI83EcxSkYSALedz32e09Eqvk6KG9R1UTgVWAfcBBIUdVvfBtVsWqgqgddrw8BDTxxUksq5oKJSHXgC+BhVT3p63i8RUSGA0dUdb2vYylmAUBX4G1V7QKcwkOPREoiVxvCSJxk2hioJiJ3+jYq31BnbIlHxpdYUrl4iUAzt/dNXWVlmohUwEkoH6vql76Ox8v6ACNEJB7n8ebVIvIf34ZULBKABFXNr4XOwUkyZdU1wB5VTVLVbOBLoLePYypOh0WkEYDr3yOeOKkllYu3DggRkWARqYjTsDffxzF5lYgIznP2bar6D1/H422q+qSqNlXVljg/3x9Utcx/glXVQ8B+EWnrKhoIxPgwJG/bB/QUkaqu/+MDKcMdE85iPjDW9XosMM8TJw3wxEnKE1XNEZEHgCU4vUVmqGq0j8Pytj7AXcAWEdnoKntKVRf5LiTjJQ8CH7s+MMUB430cj9eo6hoRmQNswOnh+AtldLoWEfkU6A8EikgC8AwwBfhMRH6DswTIrR65lk3TYowxxlPs8ZcxxhiPsaRijDHGYyypGGOM8RhLKsYYYzzGkooxxhiPsaRizGUQkZ9c/7YUkTEePvdTZ7uWMSWZdSk2xgNEpD/wqKoOv4hjAlQ1p4jtaapa3QPhGVNsrKZizGUQkTTXyylAXxHZ6Fqjw19EXhGRdSKyWUTude3fX0RWish8XKPVRWSuiKx3resxyVU2BWf23I0i8rH7tcTximsNkC0icpvbuZe5rYfysWukOCIyxbUezmYRebU4v0emfLER9cZ4xhO41VRcySFFVbuLSCXgRxHJnwG3KxCmqntc7+9R1WMiUgVYJyJfqOoTIvKAqkac5Vo3ARE4650Euo5Z4drWBeiIM4X7j0AfEdkG3Ai0U1UVkdqevXVj/sdqKsZ4x7XA3a5pbdYA9YAQ17a1bgkFYLKIbAJ+xpmsNISiXQl8qqq5qnoYWA50dzt3gqrmARuBlkAKkAlMF5GbgPTLvDdjzsmSijHeIcCDqhrh+gp2W6vjVMFOTlvMNUAvVe2MM//U5Sxpe9rtdS6Q327TA2fW4eHA4ss4vzFFsqRijGekAjXc3i8B7nctGYCIhJ5jwatawHFVTReRdjjLNefLzj++kJXAba52myCc1RrXnisw1zo4tVwTgD6C89jMGK+wNhVjPGMzkOt6jDUTZ633lsAGV2N5EmdfrnUxcJ+r3WMHziOwfFOBzSKyQVXvcCv/CugFbMJZWOmPqnrIlZTOpgYwT0Qq49Sgfn9Jd2jMBbAuxcYYYzzGHn8ZY4zxGEsqxhhjPMaSijHGGI+xpGKMMcZjLKkYY4zxGEsqxhhjPMaSijHGGI/5f6zsJawga5ZlAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for strategy in strategies:\n", + " plt.plot(iters[strategy.name], auc_history[strategy.name], label=strategy.name)\n", + "plt.title('AUC')\n", + "plt.xlabel('iterations')\n", + "plt.ylabel('AUC')\n", + "plt.legend()\n", + "plt.show() " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABAbUlEQVR4nO3dd3gU5fbA8e9JIIReAwqBhA5JgAChKr0qVUFBOijY8dr7tV7F8rNcRQUVQUSqgrmAIioISEtCr9IChBpaKAmp5/fHLjHA0rPZlPN5njzszDvvzNmEzcnMO3NeUVWMMcaYC3l5OgBjjDHZkyUIY4wxLlmCMMYY45IlCGOMMS5ZgjDGGOOSJQhjjDEuWYIwxhjjkiUIY26AiPQTkUgROS0iB0TkZxG51dkWJCLhIhInIqdEZIGINHe2NRWRMyJSxMU+V4vII1n9Xoy5kCUIY66TiDwBfAS8BZQDKgGfAT1EpCrwF7AeqAyUB2YCv4pIM1VdDsQAvS/YZwgQBEzOordhzCWJPUltzLUTkeLAPmCoqk530T4RKK2qt1+w/nMgWFVbisgLQHtVbZuh/V2guqre4d53YMyV2RmEMdenGeCL46zAlQ7ARYkDmAbcIiIFgYlASxGpCCAiXkA/YELmh2vMtbMEYcz1KQ0cUdWUS7SXAQ64WH8Ax+eulKruBRYCA51t7YACwJzMDdWY62MJwpjrcxQoIyL5LtF+BLjZxfqbgTTguHN5Av8kiIHAFFVNzsxAjbleliCMuT7LgESg5yXafwPucrH+bmCZqsY7l38E/EWkDXAndnnJZCOX+uvHGHMZqhonIv8GRotICvArkAy0B9oArwERIvIf4P+cbUOAQUDHDPs5IyIzgG+A3aoamaVvxJjLsDMIY66Tqv4f8ATwEhAL7AUeAWap6jbgVqAeEI1j7KEX0ElV/7pgVxOAAODbrIncmKtjt7kaY4xxyc4gjDHGuGQJwhhjjEuWIIwxxrhkCcIYY4xLueY21zJlymhgYKCnwzDGmBwlKirqiKr6uWrLNQkiMDCQyEi7hdwYY66FiOy+VJtdYjLGGOOSJQhjjDEuuTVBiEhnEdkqIttF5DkX7Q+IyHoRWSMiS0QkyLk+UEQSnOvXiMgX7ozTGGPMxdw2BiEi3sBoHHXxY3DUpQlX1U0ZNvteVb9wbt8d+ADo7GzboaqhNxJDcnIyMTExnD179kZ2Y4y5Dr6+vvj7+5M/f35Ph2KukzsHqRsD21V1J4CITAF6AOkJQlVPZti+MJCpdT9iYmIoWrQogYGBiEhm7toYcxmqytGjR4mJiaFy5cqeDsdcJ3deYqqAo3jZOTHOdecRkYdFZAfwLjAyQ1Nl5+Ttf4pIC1cHEJERzgnjI2NjYy9qP3v2LKVLl7bkYEwWExFKly5tZ+85nMcHqVV1tKpWBZ7FURUTHJUvK6lqfRzVMr8XkWIu+o5V1TBVDfPzc3kbryUHYzzEPns5nzsTxD6gYoZlf+e6S5mCc/IVVU1U1aPO11HADqCGe8I0xpgcKi0VNvwIUePdsnt3JogIoLqIVBYRH6AvEJ5xAxGpnmGxC7DNud7POciNiFQBqgM73Rir24gIAwYMSF9OSUnBz8+Prl27Xtf+wsPDGTVqVGaFd81at25NzZo1CQ0NJTQ0lBkzZnDfffexadOmy/bJzIcYAwMDOXLkyGW32bJlC6GhodSvX58dO3Zk2rEvZ82aNcydOzd9OTN/Vv/5z38IDg6mbt26hIaGsmLFCgA++ugj4uPjr9D7YuPHj2f//v2ZEpvxgLRUWDcdPmsGM4bC6u/ADVM3uG2QWlVTROQRYB7gDYxT1Y0i8joQqarhwCMi0h7HbFvHgcHO7i2B10UkGcf8vQ+o6jF3xepOhQsXZsOGDSQkJFCwYEHmz59PhQoXDcVcte7du9O9e/dMjPDaTZo0ibCwsPTl3r17u+1YKSkp5Mt37f9NZ82aRe/evXnppZeuvDGOQVVVxcvr+v9mWrNmDZGRkdx+++1A5v2sli1bxuzZs1m1ahUFChTgyJEjJCUlAY4EMWDAAAoVKnRRv9TUVLy9vV3uc/z48YSEhFC+fPkbjs9kodQUWD8NFr0Px3ZA2SDoPQ6CeoIbLum5dQxCVeeqag1Vraqq/3Gu+7czOaCqj6lqsKqGqmobVd3oXP9DhvUNVPV/7ozT3W6//XbmzJkDwOTJk7nnnnvS21auXEmzZs2oX78+zZs3Z+vWrQB8+OGHDBs2DID169cTEhJCfHw848eP55FHHgFgyJAhPPjggzRt2pQqVaqwcOFChg0bRu3atRkyZEj6MYoUKZL+esaMGeltV9v/Ss6dIaSmpjJkyBBCQkKoU6cOH374Yfo206dPp3HjxtSoUYPFixcDjl9gTz/9NI0aNaJu3bqMGTMGgIULF9KiRQu6d+9OUFDQJY8bHR1N7dq1GT58OMHBwXTs2JGEhATmzp3LRx99xOeff06bNm0A+OCDDwgJCSEkJISPPvoovX/NmjUZNGgQISEhLF68mFq1ajFkyBBq1KhB//79+e2337jllluoXr06K1euvOTPLCkpiX//+99MnTqV0NBQpk6det7PKjo6mrZt21K3bl3atWvHnj170n8GI0eOpHnz5lSpUoUZM2Zc9D4PHDhAmTJlKFCgAABlypShfPny/Pe//2X//v20adMm/X0WKVKEJ598knr16rFs2TJef/11GjVqREhICCNGjEBVmTFjBpGRkfTv35/Q0FASEhKIioqiVatWNGzYkE6dOnHgwAEAIiIi0s9ann76aUJCQgBo2bIla9asSY/x1ltvZe3atVf9f8Zco5QkWPUtfNoQZj0IPoXg7onwwF8Q0gsVN/0qP/eXU07/atiwoV5o06ZN6a9fDd+gd3+xNFO/Xg3fcNExL1S4cGFdu3at9urVSxMSErRevXq6YMEC7dKli6qqxsXFaXJysqqqzp8/X++8805VVU1NTdUWLVrojz/+qA0bNtQlS5aoquo333yjDz/8sKqqDh48WPv06aNpaWk6a9YsLVq0qK5bt05TU1O1QYMGunr16vQYzpk+fboOHjz4mvpn1KpVK61Ro4bWq1dP69Wrp0eOHNFWrVppRESERkZGavv27dO3PX78eHqfJ554QlVV58yZo+3atVNV1TFjxugbb7yhqqpnz57Vhg0b6s6dO3XBggVaqFAh3blzp8vvaUBAgMbGxuquXbvU29s7Pc677rpLJ06cqKqqr7zyir733nuqqhoZGakhISF6+vRpPXXqlAYFBemqVat0165dKiK6bNkyVdX0/WX8HgwdOjT9+9OjR4/L/swy/mwuXO7atauOHz9eVVW//vrr9H0NHjxYe/furampqbpx40atWrXqRe/31KlTWq9ePa1evbo++OCDunDhwou+F+cAOnXq1PTlo0ePpr8eMGCAhoeHp/9MIiIiVFU1KSlJmzVrpocPH1ZV1SlTpujQoUNVVTU4OFiXLl2qqqrPPvusBgcHq6rq+PHj9bHHHlNV1a1bt6qrz5/q+Z9Bcx2Sz6qu/Er1g2DVV4qpjmmlumWualqaqqpu3Benj01epf+asvq6D4Hjio7L36u5plhfdla3bl2io6OZPHly+uWHc+Li4hg8eDDbtm1DREhOTgbAy8uL8ePHU7duXe6//35uueUWl/vu1q0bIkKdOnUoV64cderUASA4OJjo6GhCQ0MvG9v19L/wEtM5VapUYefOnTz66KN06dKFjh07prfdeeedADRs2JDo6GgAfv31V9atW5f+V3NcXBzbtm3Dx8eHxo0bX9X985UrV06PMeO+M1qyZAl33HEHhQsXTo9l8eLFdO/enYCAAJo2bXre/jJ+D9q1a5f+/Tm370v9zC5n2bJl/PjjjwAMHDiQZ555Jr2tZ8+eeHl5ERQUxKFDhy7qW6RIEaKioli8eDELFiygT58+jBo1yuVZnre3N7169UpfXrBgAe+++y7x8fEcO3aM4OBgunXrdl6frVu3smHDBjp06AA4zuxuvvlmTpw4walTp2jWrBkA/fr1Y/bs2QDcddddvPHGG7z33nuMGzfums44zVVIPus4Y1jyIZzaD/6NoOuHUK09Cvy1/ShjFu1g8bYjFPbxZkDTAFQ10+8cyzMJ4pVuwR49fvfu3XnqqadYuHAhR48eTV//8ssv06ZNG2bOnEl0dDStW7dOb9u2bRtFihS57GDiucsOXl5e6a/PLaekpADn32544X3pV9P/apUsWZK1a9cyb948vvjiC6ZNm8a4cePOO463t3f6flWVTz75hE6dOp23n4ULF6b/Mr+SjDF7e3uTkJBwTTFfeJwLvwcZvz/n4r7cz+x6ZDymXmKg0dvbm9atW9O6dWvq1KnDhAkTXP5S9vX1TR93OHv2LA899BCRkZFUrFiRV1991eVzCapKcHAwy5YtO2/9iRMnLhlzoUKF6NChAz/99BPTpk0jKirqKt6puaKkeIj6Bv76GE4fgkrNoednUKU1KWnKnLX7GbtoJxv3n8SvaAGe6VyT/o0DKF7IPU+re/w5iLxi2LBhvPLKK+l/nZ4TFxeXPmg9fvz489aPHDmSRYsWcfToUZfXpq9WuXLl2Lx5M2lpacycOfO693MlR44cIS0tjV69evHmm2+yatWqy27fqVMnPv/88/S/wP/++2/OnDmT6XG1aNGCWbNmER8fz5kzZ5g5cyYtWrh89vKqXOpnVrRoUU6dOuWyT/PmzZkyZQrgOAO7luNv3bqVbdu2pS+vWbOGgICAKx7zXDIoU6YMp0+fPu//UMZ+NWvWJDY2Nj1BJCcns3HjRkqUKEHRokXT75g6F/859913HyNHjqRRo0aULFnyqt+PcSHxNCz5CD6uC/NeAL+aMGQODPuZ+Iot+GZpNK3fX8hjU9aQkJzKO73qsOTZNjzUuprbkgPkoTMIT/P392fkyJEXrX/mmWcYPHgwb775Jl26dElf//jjj/Pwww9To0YNvv76a9q0aUPLli2v69ijRo2ia9eu+Pn5ERYWxunTp6/7fVzOvn37GDp0KGlpaQC8/fbbl93+vvvuIzo6mgYNGqCq+Pn5MWvWrEyPq0GDBgwZMoTGjRunH7d+/fouL0ddjUv9zNq0acOoUaMIDQ3l+eefP6/PJ598wtChQ3nvvffw8/Pjm2++uerjnT59mkcffZQTJ06QL18+qlWrxtixYwEYMWIEnTt3pnz58ixYsOC8fiVKlGD48OGEhIRw00030ahRo/S2IUOG8MADD1CwYEGWLVvGjBkzGDlyJHFxcaSkpPCvf/2L4OBgvv76a4YPH46XlxetWrWiePHi6fto2LAhxYoVY+jQodf0/TMZnD0JK8fCstGQcAyqtoWWz0BAM46cTmTCr1v5dtlu4hKSCQsoySvdgmlXqyxeXlnzEKJc6pQ2pwkLC9ML77XfvHkztWvX9lBExuR8p0+fTr8LbtSoURw4cICPP/4YgP3799O6dWu2bNlyyduD7TN4CQknYMUYWD4azsZB9U7Q6hnwD2PXkTN8uXgnM6JiSE5No0PtctzfqgoNA0q5JRQRiVLViwcVsTMIY8xlzJkzh7fffpuUlBQCAgLSL6l9++23vPjii3zwwQc39OxInhN/DJZ/5kgOiSehZhdo9TSUr8+qPccZOzGKeZsOkt/bi14NKnBfiypU9Sty5f26iZ1BGGPcxj6DTmeOwLJPYeWXkHQagnpAy6dJKxvCH1sOM3bRTlZGH6OYbz4GNgtgcPNAyhb1zZLQ7AzCGGM84dQhWPpfiBwHyQkQ0gtaPkViqRr8tHo/Y79fxPbDp6lQoiAvdw2iT6OKFCmQfX4tZ59IjDEmtzi533GratR4SE2COndDiyeJK1KZ71fs4Zu/FnD4VCK1by7GR31C6VL3ZvJ7Z79LdZYgjDEms5zYC3995HjITdOgXl+49QkO5CvPuCW7mLzyD04npnBrtTK8f1c9WlQvk63LoluCMMaYG5GWCnuWwbqpsGayY139/nDr42xJLMXY33cSvmYBCnSpczMjWlYhpELxy+4yu8h+5zS5TMZCeXPnzqVGjRrs3r2bV199lffff/+yfa2stZW1NtlUShJs+w3CH4X3a8D4LrBuGjQcgo5cxdKglxky6zCdP1rMz+sPMqBpAAufas1/76mfY5ID2BlElvn9998ZOXIk8+bNS38KNjNYWWsra22ySNIZ2P47bP4f/P2L4zZVn6JQoxPU7kZKlbb8su00Y7/bybqYOEoX9uHJDjUY0DSAkoV9PB39dbEziCywaNEihg8fzuzZs6lateo197ey1lbW2nhIwglYOxWm9Id3q8K0gbD9NwjqDv2mwdPbievyBV8dq0vbT6J45PvVnDqbwn/uCOGv59ryaLvqOTY5AHmn3LfOfVZ13O2Z+zX32SuW0s2XL5+WLFlS165de976jOWoL8XKWuecstbGtRxZ7vvUYdXIb1Qn3qn6WmlHme33a6rOeUp155+qKY7PxJYDJ/X5H9dprZd+1oBnZ2uvz/7Sn9fv15TUNM/Gf42wct+ekz9/fpo3b87XX3+dXqLgelhZ6/NZWWuTqeJiHJeONv/PMeCsaVAyEJo+CLW7Q4WG4OVFSmoav20+zISl0SzbeRSffF70qFeewc0Dc9TYwtXKOwniNs/M4+zl5cW0adNo164db731Fi+88MJ17cfKWp/PylqbG3ZkO2wOdySF/c7Kw2WDoOXTjqRQLjh9Gs/jZ5KYErGX75bvZt+JBMoX9+WZzjXp26gSpXLyJaQryDsJwoMKFSrEnDlzaNGiBeXKlePee+91y3FatGjBkCFDeO6551BVZs6cycSJE697fzdS1nrgwIHXVdbay8uL6tWrA67LWpcpU+aifq7KWp+bJ/tSZa2bNWtGcnIyf//9N8HBwellrZs0aeKyrHW3bt1o0aKFlbXOyVTh4Pp/zhRiNzvWV2gI7V+FWt2gTLXzumzYF8e3y6L5ac1+ElPSaFalNC93rU372uXIlw0fbMtsliCySKlSpfjll19o2bIlfn5+ALz55pvpA8kAMTExN3QMK2ttZa3NBdLSYF/kP2cKx6NBvBwT8XR+B2p3heL+53VJTk3jlw0HmbA0msjdxymY35teDf0Z3CyQmjcV9cz78BC3FusTkc7Ax4A38JWqjrqg/QHgYSAVOA2MUNVNzrbngXudbSNVdd7ljmXF+syNuNGy1sY1j3wGU1Ng9xLnmcJsOH0QvPJDldZQuxvUvB2K+F3ULfZUIpNX7mHSit0cOplIpVKFGNQsgLsaVnTrpDye5pFifSLiDYwGOgAxQISIhJ9LAE7fq+oXzu27Ax8AnUUkCOgLBAPlgd9EpIaqprorXpO3WVnrXODkflj6Kaz9HhKOQ76CUL29YzyhekcoWMJltzV7TzBhaTRz1h0gKTWNFtXL8NYddWhdsyzeWTQxT3blzktMjYHtqroTQESmAD2A9AShqiczbF8YOHc60wOYoqqJwC4R2e7c3/mji8Zkkj59+tCnT5+L1g8aNIhBgwZ5ICJz1Y7vdtQ/Wv2do+xFUA8IuROqtgOfix+sBEhMSWXu+gOMX7qbtXtPUNjHm3saV2RQ80CPzr+Q3bgzQVQA9mZYjgGaXLiRiDwMPAH4AG0z9F1+Qd8KLvqOAEYAVKpUyWUQqpqti2EZk1u58/I14LgLackHjhpICNQfALc8BqUqX7LLoZNnmbR8N9+v3MOR00lU8SvMa92DubNBBYr65t7LSNfL44PUqjoaGC0i/YCXgMHX0HcsMBYcYxAXtvv6+nL06FFKly5tScKYLKSqHD16FF9fN0x6c2gjLP4/2DgTvH2g0XBo/igUv+hvyPRYonYfZ/zSaH7ZcJBUVdrWLMvg5oHcWq1Mls3vnBO5M0HsAypmWPZ3rruUKcDn19nXJX9/f2JiYoiNjb3WrsaYG+Tr64u/v/+VN7xa+1Y5EsOW2eBTBJqPhGYPQ5GyLjc/m5xK+Nr9TFgazcb9Jynqm48hzQMZ2CyAgNKFXfYx53NngogAqotIZRy/3PsC/TJuICLVVXWbc7ELcO51OPC9iHyAY5C6OrDyWgPInz8/lStf+nTTGJMD7F4Gi9931EDyLQ6tnoMm90OhUi4333cigYnLdjM1Yg/H45OpUa4I/7kjhDvqV6CQj8cvmuQobvtuqWqKiDwCzMNxm+s4Vd0oIq/jqP0RDjwiIu2BZOA4zstLzu2m4RjQTgEetjuYjMlDVGHnQlj0vuOW1UKlod0r0Og+8C120eZpacri7UeYtHw3v212lHHpGHQTg5oH0KyKXWK+Xm59DiIruXoOwhiTw6jC3/Ng0XuOB9yK3uy4lNRwMPhcfFno6OlEpkfF8P2KPew5Fk/pwj7c3agiA5oGUKFEQQ+8gZzHI89BGGPMVUtLczztvPh9RzmMEpWgywcQ2h/ynz/Qraqs3HWMSSv28MuGgySlptGkcime6lSTTsHlKJDP9dwg5tpZgjDGeE5qCmz4wTH4fGQrlK4GPT+HOneB9/m3ncYlJDNzVQyTVuxh2+HTFPXNR/+mlejfpBLVyuatEhhZxRKEMSbrpSTB2smO5xiORzuqqPYeB0E9wev8M4B1MSeYtHwP4Wv3k5CcSr2KJXi3d1261S1PQR87W3AnSxDGmKyTnACrJjqefD65D8rXh05vQY3bIEMpk/ikFMLX7GfSij2s3xdHIR9vetavQP8mlXLlvAvZlSUIY4z7JZ6CyHGOWklnDkPFptD9v45yGBnuMNp68BTfr9jNj6v2cSoxhZrlivJGj2B61K9AMXvSOctZgjDGuE/CCVg5FpZ/5iigV6U1tPwGAm5JTwyJKan8vP4gk1bsJiL6OD75vOhS52b6N6lEw4CSdouqB1mCMMZkvjNHHElh5ZeQeBJqdIYWT0HFf+bqiD5yhskr9zA9KoZjZ5IILF2IF2+vTa+G/rl6lracxBKEMSbznDoISz9xXE5KToCg7tDiSbi5HkD6nM6TVuxm8bYjeHsJHYPK0b9JAM2rlra6SNmMJQhjzI07sRf++hhWfQtpyY7bVG99AsrWAmD/iQSmROxlasQeDp1MpHxxX57sUIO7G1WkXDE3FPQzmcIShDHm+h3dAUs+dNyyikDoPXDr41CqCmlpyqKth5m0Yg+/bz6EAq1r+PGfngG0rumXJ+Z0zuksQRhjrt3hLY6H2zbMcEznGTbMURKjREXi4pOZ/OcOJq3Yzd5jCZQp4sODravSt1ElKpZyPYGPyZ4sQRhjrt6BdY46SZv/B/kLQtOHHHMxFL2J/ScSGDd7E5NX7uFMUipNq5Ti2c616Bh0Ez757GwhJ7IEYYy5sphIR2L4+xcoUMwx8Nz0IShcmi0HTzL25zWEr9mPAt3rlWd4iyoElb+46qrJWSxBGGMuLXqJIzHsXAgFS0Kbl6DxcNS3OMt3HmPMopUs3BpLIR9vBjULZNitgfiXtMtIuYUlCGPM+VRhxx+OuRj2LIXCZaHDGxA2jNT8hZm38SBj/tzA2pg4yhTx4amONRjQNIAShezZhdzGEoQxxkEVtv7sOGPYvwqKVYDb3oUGgziLDzOiYvhycQS7j8YTWLoQ/7kjhF4N/PHNbwXzcitLEMbkdWmpsOknx11JhzZAiQDo9jHUu4cTScLERbsZvzSao2eSqFexBM/fVosOQTfhbQ+15XqWIIzJq1JTYP10R2I4ug1KV4c7xkBIb/bGJfH13O1MjdhLQnIqbWuV5f6WVWhcuZTVRspDLEEYk9ekJDoebFv8AZzYDeVC4K7xULs7Gw+eZuz09cxedwABeoRWYETLKtS8ySbkyYvcmiBEpDPwMeANfKWqoy5ofwK4D0gBYoFhqrrb2ZYKrHduukdVu7szVmNyveQERymMvz52zsXQADqPQmt0YunO43zxTSSLtx2hSIF8DLslkGG3Vubm4javc17mtgQhIt7AaKADEANEiEi4qm7KsNlqIExV40XkQeBdoI+zLUFVQ90VnzF5RuJp51wMnzjmYqjUDLp/Qkpga+ZuPMTY0UvZsO8kfkUL8GznWvRrUoniBW3uBePeM4jGwHZV3QkgIlOAHkB6glDVBRm2Xw4McGM8xuQtZ+NgxVhYPto5F0MbaDme+PJNmB4Zw5c//EnM8QSq+BXmnV516Fm/AgXy2R1J5h/uTBAVgL0ZlmOAJpfZ/l7g5wzLviISiePy0yhVnXVhBxEZAYwAqFSp0o3Ga0zukHACln/u+EqMS5+L4WjJukxYtpuJ3/7B8fhkGgaU5N9dg2hfu5yV2TYuZYtBahEZAIQBrTKsDlDVfSJSBfhDRNar6o6M/VR1LDAWICwsTLMsYGOyo/hjjqSw4gvHJD21ukKrZ9jtU42vFu9iWuQfJKak0SGoHPe3rEJYYClPR2yyOXcmiH1AxQzL/s515xGR9sCLQCtVTTy3XlX3Of/dKSILgfrAjgv7G5PnxR+DZaNhxRhIOgW1u0GrZ9kmgXz0+zZ+Xr8Qby/hzvr+DG9ZmWpl7Y4kc3XcmSAigOoiUhlHYugL9Mu4gYjUB8YAnVX1cIb1JYF4VU0UkTLALTgGsI0x55w5Css+dcz5nHQagnpAy2c4WaImH83fxoRliymU35sRLasy9JZAm5jHXDO3JQhVTRGRR4B5OG5zHaeqG0XkdSBSVcOB94AiwHTnwzfnbmetDYwRkTTAC8cYxCaXBzImrzlzxHFH0sovITkegntCy2dI86vNjKgY3v1qIUfPJHFP40o82aEGpYsU8HTEJocS1dxx6T4sLEwjIyM9HYYx7nM6Fpb+FyK+diSGkDuh5TNQthar9xzn1fCNrI2Jo2FASV7rHkxIheKejtjkACISpaphrtqyxSC1MeYyTh92PNwWOQ5SzkJIL2j5NPjV5PCps7w7fS0zomIoW7QAH/UJpUdoeSuHYTKFJQhjsqtTh/5JDKmJUOcuR2IoU52klDQmLNrJx79vIzEllQdaVeWRttUoUsA+0ibz2P8mY7KbUwdhyUcQ9Q2kJkPdu6HFU1CmGgB//h3La//byM7YM7Sp6ce/uwVTuUxhz8ZsciVLEMZkFyf3OxPDeEhLgXp9HVN7lq4KwJ6j8bwxZxPzNx0isHQhxg0Jo22tch4N2eRuliCM8bS4ffDXRxA1ATT1n8RQqgoA8UkpfLZgB2MX7ySfl/Bs51oMuzXQymIYt7MEYYynxMXAkg8dFVY1DUL7ORJDyUAAVJX/rTvA23M3cyDuLHfUr8Bzt9Wy5xlMlrEEYUxWO7EXlnwAqyY6luv3h1ufgJIB6Zts2n+SV/+3kZW7jhFcvhif3FPfSmOYLGcJwpiscny3IzGsnuRYbjDQkRhK/FOR5kR8Ev/3699MWrGb4gXz89YddejTqKJN72k8whKEMe52fDcsfh/WfA/iBQ0Hw62PQ3H/9E1S05TJK/fw/q9bOZmQzMCmATzeoQYlCvl4MHCT11mCMMZdUpMdJTEWOidSDBsGt/wLilc4b7OVu47xavhGNh04SZPKpXi1ezC1by6W9fEacwFLEMa4w/7VEP4oHFwPtbtD51EXJYaDcWd5a+5mwtfup3xxXz7tV58udW62p6BNtmEJwpjMlBQPC99ylN8uXBb6fOcov51BYkoqXy3exegF20lJU0a2rcYDratSyMc+jiZ7sf+RxmSWnQvhf4/B8WhoOBTavwoFS6Q3qyp/bDnM67M3sftoPB2DyvFy1yAqlirkoYCNuTxLEMbcqPhj8OvLsOY7KFUVhsyBwFvP22RH7GnemL2JhVtjqepXmG+HNaZlDT8PBWzM1bEEYcz1UoVNs2DuMxB/1HHLaqtnIH/B9E3ik1L4+PdtjFuyiwL5vHmpS20GNw8kv7eX5+I25ipZgjDmesTtg7lPwda5cHMoDPwRbqpz3ia/bTrEK+Eb2Xcigd4N/Xmmc03KFrWnoE3OYQnCmGuRlgZR42D+q46Ceh3fhCYPgvc/H6X9JxJ4NXwjv246RI1yRZj+QDMa2VPQJgeyBGHM1Yr9G/43EvYsgyqtoetHUKpyenNKahrjl0bzwfy/SVPl2c61uPfWyvjks8tJJmeyBGHMlaQkwdKP4c93IX8h6PGZo7BehucVVu85zgszN7D5wEna1irLa92D7e4kk+NZgjDmcmKiHA+8Hd4IwXfCbe9AkbLpzXEJybz7yxa+X7mHckV9+WJAAzoF32QPu5lcwa0JQkQ6Ax8D3sBXqjrqgvYngPuAFCAWGKaqu51tg4GXnJu+qaoT3BmrMedJOgN//AdWfA5FboK+k6HW7enNqkr42v28MXszx84kMrR5ZZ7oWMOm/DS5itv+N4uINzAa6ADEABEiEq6qmzJsthoIU9V4EXkQeBfoIyKlgFeAMECBKGff4+6K15h023+D2Y/DiT0Qdi+0fwV8i6c37zpyhpdnbWDJ9iPU8y/O+KGNCKlQ/DI7NCZncuefO42B7aq6E0BEpgA9gPQEoaoLMmy/HBjgfN0JmK+qx5x95wOdgclujNfkdfHH4JfnYd0UKF0dhv4CAc3SmxNTUvl84Q4+W7iDAt5evNEjmH5NAqwUt8m13JkgKgB7MyzHAE0us/29wM+X6Vvhwg4iMgIYAVCpUqUbidXkZaqw4Qf4+Vk4ewJaPg0tnoL8/zyzsHT7EV6atYGdR87QrV55Xu5Sm7I2s5vJ5bLFBVMRGYDjclKra+mnqmOBsQBhYWHqhtBMbndiL8x5Arb9ChUaQvdwKBec3hx7KpG35m5m5up9BJQuxIRhjWllJTJMHuHOBLEPqJhh2d+57jwi0h54EWilqokZ+ra+oO9Ct0Rp8qa0NIj4Cn5/zTEfdKe3ocn94OXtbFamROxl1M+bSUhOZWTbajzUphq++b09HLgxWcedCSICqC4ilXH8wu8L9Mu4gYjUB8YAnVX1cIamecBbIlLSudwReN6NsZq85PAWx62rMSuhalvHA28Z5oPefOAkL85cz6o9J2hapRRv9qxDtbJFPBevMR7itgShqiki8giOX/bewDhV3SgirwORqhoOvAcUAaY77xvfo6rdVfWYiLyBI8kAvH5uwNqY65aSCEs+hEXvQ4EicMcYqNsn/YG3M4mOwnpfL9lFiYL5+eDuetxRv4I902DyLFHNHZfuw8LCNDIy0tNhmOzq4Hr44T6I3QIhvR0zvBX5Zyxh/qZDvPLTBvbHneWexhV5tnMtmw/a5AkiEqWqYa7assUgtTFutXaqYyIf3+LQbxrU6JTetM9ZWG/+pkPULFeUGffUJ8wK6xkDWIIwuVlKEsx7ASK+hIBb4a5v0stkpKSm8c1f0Xz429+owvO31WLYrZVtngZjMrhsghCRm1T1YFYFY0ymObkfpg2CmAho/ii0ezW9JPeqPcd50VlYr33tsrzaPRj/klZYz5gLXekMYo2IbMDxBPMPqnrC/SEZc4N2LYYZQyE5Ae6aAME9AYiLT+adeVuYvHIPNxXzZczAhnQMKmeD0MZcwpUSRAWgPY5bVN8SkeU4ksVPqprg7uCMuSaqsOxTmP8KlHbODe1XE4CFWw/z1PS1HI9P5t5bKvN4hxoUtsJ6xlzWZT8hqpqK4zbVeSLiA9yGI1l8JCK/q2r/LIjRmCtLPAU/PQybfoLa3aHnZ1CgKGlpymcLt/N/8/+mZrmiTBjWmODyVljPmKtx1X9CqWqSiGwCNgMNgdpui8qYaxG7FaYOgKPbocMbjjEHEU6dTebJaWv5ddMheoaW5+0761LQx56ENuZqXTFBiEhFHGcN9wCFcVxi6q6qW9wcmzFXtnGW48whny8M+gkqtwRg++FTjJgYxe6j8fy7axBDbwm0sQZjrtGV7mJaimMcYjowXFWjsiQqY64kNQV+fxWWfgL+jRyD0cUdBX9/2XCQJ6etoaCPN5Pua0LTKqU9G6sxOdSVziCeAxarqopIEREpoqqnsyIwYy7p9GGYMQyiF0Oj4dDpLcjnQ2qa8sH8rYxesIN6FUvwxYAG3Fy8oKejNSbHutIg9SIReVBEnsdxeUlE5BTwjqp+liURGpPR3gjH8w0Jxx21lOr1BeBEfBIjp6xh0d+x9G1Ukdd6BFMgn403GHMjrnSJ6UXgFqB1hpnhqgAfi0gpVX0zC2I0xnELa8RXjhnfileA++bDTXUA2LT/JPd/F8mhuETevrMO9zS2yaOMyQxXusQ0CKinqmfPrVDVnSJyN7AWsARh3C8p3jFH9LopUL0T3DkGCjoqwf+0Zh/P/rCOEgV9mHJ/UxpUKnmFnRljrtaVEoRmTA4ZViaISJqbYjLmH8d2wtSBcGgjtHnRMRWolxfJqWm8PXcL4/7aRePAUozu3wC/ogU8Ha0xucqVEsQ+EWmnqr9nXCki7YAD7gvLGGDrL/DjCMd8Df1nQPX2ABw5ncjDk1axYtcxhjQP5MUuta3InjFucKUEMRL4SUSWAOducQ3DMS7Rw52BmTwsLRX+fMfxdVNd6DMRSgYCsGbvCR78LopjZ5L4sE897qjv79lYjcnFrpQgEoEhQA3g3Ezui4AvgYsuPRlzw+KPwY/DYftvEDoAurwP+R23qk6N2MPLszZStlgBfniwOSEVrGSGMe50pQTxEfC8qo7LuFJE6jjburknLJMn7V8D0wbCqYOOeaIbDgERElNSee1/m/h+xR5aVC/Df/vWp2Rhm+3NGHe7UoIop6rrL1ypqutFJNA9IZk8afV3MPsJKOwHQ38B/4YAHIw7y4OToli95wQPtq7KUx1r4u1lJTOMyQpXShAlLtNmj6iaG5eSCD8/A1HjoXIr6D0OCpcBYOWuYzw0aRXxSSl83r8Bt9W52bOxGpPHXOnWj0gRGX7hShG5j38GrS9JRDqLyFYR2S4iz7lobykiq0QkRUR6X9CWKiJrnF/hVzqWyYFO7IVxnR3J4dYnYOBMKFwGVWX8X7vo9+Vyivrm46eHb7HkYIwHXOkM4l/ATBHpz/l3MfkAd1yuo4h4A6OBDkAMECEi4aq6KcNme3AMgj/lYhcJqhp6hfhMTrVzoaOeUmoy9JkEtbsCcDY5lRd+XM+Pq/fRvnZZPugTSjHf/J6N1Zg86kq1mA4BzUWkDRDiXD1HVf+4in03BrZnKNExBcetsekJQlWjnW320F1eoQpLPoQ/3oAyNaHPd1CmGgB7j8XzwHdRbDpwksfb1+DRttXwsvEGYzzmqiYMUtUFwIJr3HcFYG+G5RigyTX09xWRSCAFGKWqsy7cQERGACMAKlWy+jvZXkqi48G3TbMgpBd0+y8UKALAkm1HeHTyKlLSlK8Hh9G2VjnPxmqMufoZ5TwgQFX3OYsD/iEi61V1R8YNVHUsMBYgLCxMPRGkuUpJ8Y5bWLf/Bh1eh+YjQQRVZcyinbz7yxaqlS3CmIFhVC5T2NPRGmNwb4LYB1TMsOzvXHdVVHWf89+dIrIQqA/suGwnkz0lnoLv+8Luv6D7J9BgEABnElN4ZsY65qw/QJe6N/Nur7oULpCd/2YxJm9x56cxAqguIpVxJIa+QL+r6SgiJYF4VU0UkTI4Snu867ZIjfskHIfvesP+1dDrK6jjuFlt15EzjPg2kh2xp3nh9loMb1HFpgQ1JptxW4JQ1RQReQSYB3gD41R1o4i8DkSqariINAJmAiWBbiLymqoGA7WBMc7Bay8cYxCbLnEok12djoWJd8CRrY56SrW6APD75kP8a8oa8nkLE+9twi3Vyng4UGOMK6KaOy7dh4WFaWRkpKfDMOfE7YNve0BcDNzzPVRtS1qa8vHv2/j4922EVCjGFwMa4l+ykKcjNSZPE5EoVQ1z1WYXfE3mO7YLvu0O8cdh4I8Q0JwT8Uk8PnUNC7bG0quBP/+5IwTf/DYlqDHZmSUIk7li/3acOaQkwOBwqNCADfvieHBSFAfjzvJmzxD6N6lk4w3G5ACWIEzmObDOMeYgXjBkDpQLZkZUDC/OXE/JQj5Mu78Z9W1KUGNyDEsQJnPsjYBJvcCnKAz6icQSlXlj1nq+W76HZlVK80m/+pQpYlOCGpOTWIIwN27XYpjc11Gqe3A4B8SPB8csZ83eE9zfqgpPd6xJPpsS1JgcxxKEuTHb5sPUAY4pQQf9xNJD+Xh08hLOJqdaiW5jcjhLEOb6bfoJZtwLZWujA2cyNjKOd37ZQhW/InwxoCHVyhbxdITGmBtgCcJcn7VTYNaD4N+IU72+5+kfd/PLxoN0qXMz7/SuSxErmWFMjmefYnPtIr6GOU9A5ZZsb/clI77ewO6j8bzUpTb33lrZbmE1JpewBGGuzdJP4NeXoHonfg56lyfHrqGQjzeT7mtC0yqlPR2dMSYTWYIwV0cV/nwHFr5NWu2evFPoCcZM3UiDSiX4rH9Dbiru6+kIjTGZzBKEuTJVx1nDsk85G9yXIUcHsnx1DIOaBfBSlyB88tktrMbkRpYgzOWlpcHcJyFyHIdrDaT73905cfYUH9xdjzsb+Hs6OmOMG1mCMJeWmgI/PQzrprCh8lDuWN+Bm4vn58ehTQkqX8zT0Rlj3MwShHEtJQl+uBc2hzPH7z4e3tyGdrXK8sHdoRQvlN/T0RljsoAlCHOx5ASYOhC2z+cL3/t4J6YtT3SowSNtquHlZbewGpNXWIIw53POH627/+I1RjAzsQPfDAmldc2yno7MGJPFLEGYfyQcR7/rje5bzb+SHmLHTbcxe0BDKpayWd+MyYssQRiH07GkTOiBxv7Nw0kjKV7/Dn7oabO+GZOXufUGdhHpLCJbRWS7iDznor2liKwSkRQR6X1B22AR2eb8GuzOOPO8k/s5+1VnUmK3cX/KU7TuMYx3e9e15GBMHue2MwgR8QZGAx2AGCBCRMJVdVOGzfYAQ4CnLuhbCngFCAMUiHL2Pe6uePOs49Gc/rILnDnK0z4vM/LeQYRWLOHpqIwx2YA7LzE1Brar6k4AEZkC9ADSE4SqRjvb0i7o2wmYr6rHnO3zgc7AZDfGm+ckHdxMwtfd0KR4/q/sKN4cfA+lbdY3Y4yTOxNEBWBvhuUYoMkN9K2QSXEZ4NC2SAp8fyfJacr/6o7hlZ5dbNY3Y8x5cvQgtYiMAEYAVKpUycPR5BxbVv5K+blDiFdftnX+jmHNmns6JGNMNuTOPxn3ARUzLPs712VaX1Udq6phqhrm5+d33YHmJat/nUjgnH7ESXESB82mhSUHY8wluDNBRADVRaSyiPgAfYHwq+w7D+goIiVFpCTQ0bnO3IAVU9+h3l+Psjt/FQo/+BsBVYM8HZIxJhtzW4JQ1RTgERy/2DcD01R1o4i8LiLdAUSkkYjEAHcBY0Rko7PvMeANHEkmAnj93IC1uXaalsbyL0fSZPNbrCvUhIqP/0apsjakY4y5PFFVT8eQKcLCwjQyMtLTYWQ7yUlnWTN6EI3i5rGyVDcaPDiOfPl9PB2WMSabEJEoVQ1z1ZajB6nN5Z0+eZydn/Wi0dkollV6gKZD3ka87E4lY8zVsQSRSx05uIcTX/YkKGUXK+u9TrM7H/N0SMaYHMYSRC609+81eE++i/JpcWxs9QWN2/bxdEjGmBzIEkQuszXyN8rOHoIixPSYTr0GrTwdkjEmh7IEkYusmf89tZaM5IhXaej/AzWqhXg6JGNMDmYJIpdYMf19wja8yc781Sg1fBaly/l7OiRjTA5nCSKH07Q0Vox7kqYx41hbqDHVHppO4aIlPB2WMSYXsASRgyUnJbLms8E0PfEzK0t2ocFD4+0ZB2NMprEEkUOdOXWCHaN70+hsBMsqDqfp0HftGQdjTKayBJEDHTm4l+Nf9iQ4ZQcr67xCs95PeDokY0wuZAkih9m7fT1ek3rhn3ac9S2+oHH7vp4OyRiTS1mCyEH+XrUQv/CBAOzpNpXQsLYejsgYk5tZgsgh1v4+heqLRnLcqwSp/WZQs3pdT4dkjMnlbFQzB1j5w4eELHqAffkqUuD+36lkycEYkwXsDCIb07Q0ln/zDM32fsm6go2o+vAMe8bBGJNlLEFkUynJSaz6bAjNjs8hosRthD40gfw+BTwdljEmD7EEkQ3Fn45j2+jeNE5YyTL/YTQd9n/2jIMxJstZgshmjh2K4ciXdxCSvI0VwS/T7O6nPB2SMSaPsgSRjezbuRGd2ItKaUdYd8unNOk4wNMhGWPyMEsQ2cTfq/6kTPgABCW66xTqN2rv6ZCMMXmcWy9si0hnEdkqIttF5DkX7QVEZKqzfYWIBDrXB4pIgoiscX594c44PW3tgun4/3QXZ8WXU/1mU8uSgzEmG3DbGYSIeAOjgQ5ADBAhIuGquinDZvcCx1W1moj0Bd4Bzs2PuUNVQ90VX3axcuZ/abDmFaLzVabE8FmUv6mSp0MyxhjAvWcQjYHtqrpTVZOAKUCPC7bpAUxwvp4BtBMRcWNM2YampbF8/As0XvsymwrW56bHfqeMJQdjTDbizgRRAdibYTnGuc7lNqqaAsQBpZ1tlUVktYj8KSItXB1AREaISKSIRMbGxmZu9G6kaWks//IxmkaPJrJYe2o9PpcixUp6OixjjDlPdr25/gBQSVXrA08A34tIsQs3UtWxqhqmqmF+fn5ZHuT1SEtNZeVn99LswLesKN2TBo9Nw6eAr6fDMsaYi7gzQewDKmZY9neuc7mNiOQDigNHVTVRVY8CqGoUsAOo4cZYs0RKchJR/+1HkyM/svym/jR++Bu8vL09HZYxxrjkzgQRAVQXkcoi4gP0BcIv2CYcGOx83Rv4Q1VVRPycg9yISBWgOrDTjbG6XVLiWdZ93ItGcb+wrNL9NBnxqT0dbYzJ1tx2F5OqpojII8A8wBsYp6obReR1IFJVw4GvgYkish04hiOJALQEXheRZCANeEBVj7krVnc7G3+arZ/cQYOElSyv/gTN+r/i6ZCMMeaKRFU9HUOmCAsL08jISE+HcZHTJ4+z+9Me1E5cR0TIyzS560lPh2SMMelEJEpVw1y12ZPUbhR3LJaDn3WhZvI2VjUcRZPuD3g6JGOMuWqWINzk6KEYToztSuWUvay/5RPCrK6SMSaHsQThBodidnB2XHfKpx5ma9svqd/qTk+HZIwx18wSRCbbt3MzMrE7pdNOseu2idRp2tnTIRljzHWxBJGJdm9ZRcEpvfAhiQM9pxFUv6WnQzLGmOtmCSKT7Fi3lFI/9iEVL47fPYvqQY08HZIxxtwQe1IrE2yJ+A2/H3uTiA8JA2ZT2ZKDMSYXsARxgzYsCafS7H6clGLo0LlUrFbH0yEZY0ymsARxA9b+MYXq84dx2LscvsPncXNATU+HZIwxmcYSxHWKmvsNQX8+xJ58AZR4aD5lygd4OiRjjMlUliCuw8qZnxC64nG2+9Si3KO/UqLMTZ4OyRhjMp3dxXSNVkx5myZbRrHetwFVH51FoSLFPR2SMca4hSWIa7Bswos02/Upqws1J2jkDxTwLeTpkIwxxm0sQVwFTUtj+deP02zfeCKLtafeI9+T36eAp8Myxhi3sgRxBWmpqaz84n6axU5nZaluNHxoPN757NtmjMn97DfdZaSmpBD16UCanpjL8nJ9aXL/5zYLnDEmz7AEcQnJSYms+6QPjU8tYFnF+2g69D1LDsaYPMUShAtnE86w5ZNeNIxfxvJq/6LZgNc8HZIxxmQ5SxAXOHPqBLs+7UFo4hpWBL1I07uf8XRIxhjjEZYgMog7foQDn3WjdtJmIhq8TZMeD3k6JGOM8Ri3XlQXkc4islVEtovIcy7aC4jIVGf7ChEJzND2vHP9VhHp5M44AY7HHiD2045USdrKuuYf08iSgzEmj3NbghARb2A0cBsQBNwjIkEXbHYvcFxVqwEfAu84+wYBfYFgoDPwmXN/bhG7P5q4zzvgn7KHLa3HUr/TYHcdyhhjcgx3nkE0Brar6k5VTQKmAD0u2KYHMMH5egbQTkTEuX6Kqiaq6i5gu3N/me7g3u0kfdkRv9RYdnT6lrptervjMMYYk+O4M0FUAPZmWI5xrnO5jaqmAHFA6avsi4iMEJFIEYmMjY29riCLlihDrG9lYrpNJrj57de1D2OMyY1y9CC1qo4FxgKEhYXp9eyjcNEShD47L1PjMsaY3MCdZxD7gIoZlv2d61xuIyL5gOLA0avsa4wxxo3cmSAigOoiUllEfHAMOodfsE04cG5EuDfwh6qqc31f511OlYHqwEo3xmqMMeYCbrvEpKopIvIIMA/wBsap6kYReR2IVNVw4GtgoohsB47hSCI4t5sGbAJSgIdVNdVdsRpjjLmYOP5gz/nCwsI0MjLS02EYY0yOIiJRqhrmqs2qzxljjHHJEoQxxhiXLEEYY4xxyRKEMcYYl3LNILWIxAK7b2AXZYAjmRROTpHX3nNee79g7zmvuJH3HKCqfq4ack2CuFEiEnmpkfzcKq+957z2fsHec17hrvdsl5iMMca4ZAnCGGOMS5Yg/jHW0wF4QF57z3nt/YK957zCLe/ZxiCMMca4ZGcQxhhjXLIEYYwxxqU8nyBEpLOIbBWR7SLynKfjcTcRqSgiC0Rkk4hsFJHHPB1TVhERbxFZLSKzPR1LVhCREiIyQ0S2iMhmEWnm6ZjcTUQed/6/3iAik0XE19MxZTYRGScih0VkQ4Z1pURkvohsc/5bMjOOlacThIh4A6OB24Ag4B4RCfJsVG6XAjypqkFAU+DhPPCez3kM2OzpILLQx8AvqloLqEcuf+8iUgEYCYSpagiOaQb6ejYqtxgPdL5g3XPA76paHfjduXzD8nSCABoD21V1p6omAVOAHh6Oya1U9YCqrnK+PoXjl8ZF833nNiLiD3QBvvJ0LFlBRIoDLXHMuYKqJqnqCY8GlTXyAQWdM1QWAvZ7OJ5Mp6qLcMyfk1EPYILz9QSgZ2YcK68niArA3gzLMeSBX5bniEggUB9Y4eFQssJHwDNAmofjyCqVgVjgG+dlta9EpLCng3InVd0HvA/sAQ4Acar6q2ejyjLlVPWA8/VBoFxm7DSvJ4g8S0SKAD8A/1LVk56Ox51EpCtwWFWjPB1LFsoHNAA+V9X6wBky6bJDduW87t4DR3IsDxQWkQGejSrrOadtzpTnF/J6gtgHVMyw7O9cl6uJSH4cyWGSqv7o6XiywC1AdxGJxnEZsa2IfOfZkNwuBohR1XNnhzNwJIzcrD2wS1VjVTUZ+BFo7uGYssohEbkZwPnv4czYaV5PEBFAdRGpLCI+OAa0wj0ck1uJiOC4Lr1ZVT/wdDxZQVWfV1V/VQ3E8TP+Q1Vz9V+WqnoQ2CsiNZ2r2uGY4z032wM0FZFCzv/n7cjlA/MZhAODna8HAz9lxk7zZcZOcipVTRGRR4B5OO54GKeqGz0clrvdAgwE1ovIGue6F1R1rudCMm7yKDDJ+cfPTmCoh+NxK1VdISIzgFU47tZbTS4suyEik4HWQBkRiQFeAUYB00TkXhzTHtydKceyUhvGGGNcyeuXmIwxxlyCJQhjjDEuWYIwxhjjkiUIY4wxLlmCMMYY45IlCGOcRGSp899AEemXyft+wdWxjMnO7DZXYy4gIq2Bp1S16zX0yaeqKZdpP62qRTIhPGOyjJ1BGOMkIqedL0cBLURkjXN+AW8ReU9EIkRknYjc79y+tYgsFpFwnE8pi8gsEYlyzkkwwrluFI4Ko2tEZFLGY4nDe875C9aLSJ8M+16YYT6HSc6ngxGRUc75PNaJyPtZ+T0yeUuefpLamEt4jgxnEM5f9HGq2khECgB/ici5KqENgBBV3eVcHqaqx0SkIBAhIj+o6nMi8oiqhro41p1AKI75Gso4+yxyttUHgnGUrP4LuEVENgN3ALVUVUWkROa+dWP+YWcQxlxZR2CQszTJCqA0UN3ZtjJDcgAYKSJrgeU4CkFW5/JuBSaraqqqHgL+BBpl2HeMqqYBa4BAIA44C3wtIncC8Tf43oy5JEsQxlyZAI+qaqjzq3KGeQbOpG/kGLtoDzRT1Xo4agHdyJSXiRlepwLnxjka46jO2hX45Qb2b8xlWYIw5mKngKIZlucBDzrLpCMiNS4x+U5x4LiqxotILRxTup6TfK7/BRYDfZzjHH44ZoFbeanAnPN4FHcWV3wcx6UpY9zCxiCMudg6INV5qWg8jrmdA4FVzoHiWFxP6fgL8IBznGArjstM54wF1onIKlXtn2H9TKAZsBbHJC/PqOpBZ4JxpSjwk4j44jizeeK63qExV8FuczXGGOOSXWIyxhjjkiUIY4wxLlmCMMYY45IlCGOMMS5ZgjDGGOOSJQhjjDEuWYIwxhjj0v8DAqxLtnl4/34AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for strategy in strategies:\n", + " plt.plot(iters[strategy.name], cov_history[strategy.name], label=strategy.name)\n", + "plt.title('COV')\n", + "plt.xlabel('iterations')\n", + "plt.ylabel('COV')\n", + "plt.legend()\n", + "plt.show() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scripts/train.ipynb b/scripts/train.ipynb new file mode 100644 index 0000000..f61fafb --- /dev/null +++ b/scripts/train.ipynb @@ -0,0 +1,406 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append('..')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import CAT\n", + "import sys\n", + "import json\n", + "import logging\n", + "import numpy as np\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def setuplogger():\n", + " root = logging.getLogger()\n", + " root.setLevel(logging.INFO)\n", + " handler = logging.StreamHandler(sys.stdout)\n", + " handler.setLevel(logging.INFO)\n", + " formatter = logging.Formatter(\"[%(levelname)s %(asctime)s] %(message)s\")\n", + " handler.setFormatter(formatter)\n", + " root.addHandler(handler)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "setuplogger()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# choose dataset here\n", + "dataset = 'assistment'\n", + "# modify config here\n", + "config = {\n", + " 'learning_rate': 0.002,\n", + " 'batch_size': 2048,\n", + " 'num_epochs': 10,\n", + " 'num_dim': 10,\n", + " 'device': 'cpu',\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# read datasets\n", + "train_triplets = pd.read_csv(f'../data/{dataset}/train_triples.csv', encoding='utf-8').to_records(index=False)\n", + "concept_map = json.load(open(f'../data/{dataset}/concept_map.json', 'r'))\n", + "concept_map = {int(k):v for k,v in concept_map.items()}\n", + "metadata = json.load(open(f'../data/{dataset}/metadata.json', 'r'))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = CAT.dataset.TrainDataset(train_triplets, concept_map,\n", + " metadata['num_train_students'], \n", + " metadata['num_questions'], \n", + " metadata['num_concepts'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[INFO 2021-03-01 17:38:58,388] train on cpu\n", + "[INFO 2021-03-01 17:38:58,423] Epoch [1] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:38:58,446] Epoch [1] Batch [1]: loss=1.38732\n", + "[INFO 2021-03-01 17:38:58,468] Epoch [1] Batch [2]: loss=1.04031\n", + "[INFO 2021-03-01 17:38:58,494] Epoch [1] Batch [3]: loss=0.92477\n", + "[INFO 2021-03-01 17:38:58,516] Epoch [1] Batch [4]: loss=0.86667\n", + "[INFO 2021-03-01 17:38:58,537] Epoch [1] Batch [5]: loss=0.83200\n", + "[INFO 2021-03-01 17:38:58,560] Epoch [1] Batch [6]: loss=0.80876\n", + "[INFO 2021-03-01 17:38:58,582] Epoch [1] Batch [7]: loss=0.79199\n", + "[INFO 2021-03-01 17:38:58,602] Epoch [1] Batch [8]: loss=0.77949\n", + "[INFO 2021-03-01 17:38:58,622] Epoch [1] Batch [9]: loss=0.76973\n", + "[INFO 2021-03-01 17:38:58,644] Epoch [1] Batch [10]: loss=0.76185\n", + "[INFO 2021-03-01 17:38:58,665] Epoch [1] Batch [11]: loss=0.75544\n", + "[INFO 2021-03-01 17:38:58,687] Epoch [1] Batch [12]: loss=0.75013\n", + "[INFO 2021-03-01 17:38:58,710] Epoch [1] Batch [13]: loss=0.74561\n", + "[INFO 2021-03-01 17:38:58,732] Epoch [1] Batch [14]: loss=0.74171\n", + "[INFO 2021-03-01 17:38:58,752] Epoch [1] Batch [15]: loss=0.73824\n", + "[INFO 2021-03-01 17:38:58,774] Epoch [1] Batch [16]: loss=0.73530\n", + "[INFO 2021-03-01 17:38:58,799] Epoch [1] Batch [17]: loss=0.73256\n", + "[INFO 2021-03-01 17:38:58,824] Epoch [1] Batch [18]: loss=0.73021\n", + "[INFO 2021-03-01 17:38:58,847] Epoch [1] Batch [19]: loss=0.72813\n", + "[INFO 2021-03-01 17:38:58,867] Epoch [1] Batch [20]: loss=0.72625\n", + "[INFO 2021-03-01 17:38:58,890] Epoch [1] Batch [21]: loss=0.72451\n", + "[INFO 2021-03-01 17:38:58,913] Epoch [1] Batch [22]: loss=0.72296\n", + "[INFO 2021-03-01 17:38:58,936] Epoch [1] Batch [23]: loss=0.72146\n", + "[INFO 2021-03-01 17:38:58,952] Epoch [1] Batch [24]: loss=0.72014\n", + "[INFO 2021-03-01 17:38:58,975] Epoch [2] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:38:58,998] Epoch [2] Batch [1]: loss=1.37730\n", + "[INFO 2021-03-01 17:38:59,020] Epoch [2] Batch [2]: loss=1.03269\n", + "[INFO 2021-03-01 17:38:59,041] Epoch [2] Batch [3]: loss=0.91729\n", + "[INFO 2021-03-01 17:38:59,062] Epoch [2] Batch [4]: loss=0.85982\n", + "[INFO 2021-03-01 17:38:59,082] Epoch [2] Batch [5]: loss=0.82507\n", + "[INFO 2021-03-01 17:38:59,104] Epoch [2] Batch [6]: loss=0.80209\n", + "[INFO 2021-03-01 17:38:59,125] Epoch [2] Batch [7]: loss=0.78560\n", + "[INFO 2021-03-01 17:38:59,147] Epoch [2] Batch [8]: loss=0.77328\n", + "[INFO 2021-03-01 17:38:59,169] Epoch [2] Batch [9]: loss=0.76364\n", + "[INFO 2021-03-01 17:38:59,190] Epoch [2] Batch [10]: loss=0.75607\n", + "[INFO 2021-03-01 17:38:59,212] Epoch [2] Batch [11]: loss=0.74966\n", + "[INFO 2021-03-01 17:38:59,233] Epoch [2] Batch [12]: loss=0.74428\n", + "[INFO 2021-03-01 17:38:59,257] Epoch [2] Batch [13]: loss=0.73974\n", + "[INFO 2021-03-01 17:38:59,280] Epoch [2] Batch [14]: loss=0.73591\n", + "[INFO 2021-03-01 17:38:59,301] Epoch [2] Batch [15]: loss=0.73249\n", + "[INFO 2021-03-01 17:38:59,322] Epoch [2] Batch [16]: loss=0.72959\n", + "[INFO 2021-03-01 17:38:59,344] Epoch [2] Batch [17]: loss=0.72700\n", + "[INFO 2021-03-01 17:38:59,366] Epoch [2] Batch [18]: loss=0.72460\n", + "[INFO 2021-03-01 17:38:59,387] Epoch [2] Batch [19]: loss=0.72246\n", + "[INFO 2021-03-01 17:38:59,408] Epoch [2] Batch [20]: loss=0.72059\n", + "[INFO 2021-03-01 17:38:59,430] Epoch [2] Batch [21]: loss=0.71885\n", + "[INFO 2021-03-01 17:38:59,490] Epoch [2] Batch [22]: loss=0.71727\n", + "[INFO 2021-03-01 17:38:59,513] Epoch [2] Batch [23]: loss=0.71580\n", + "[INFO 2021-03-01 17:38:59,528] Epoch [2] Batch [24]: loss=0.71453\n", + "[INFO 2021-03-01 17:38:59,551] Epoch [3] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:38:59,573] Epoch [3] Batch [1]: loss=1.36596\n", + "[INFO 2021-03-01 17:38:59,595] Epoch [3] Batch [2]: loss=1.02404\n", + "[INFO 2021-03-01 17:38:59,616] Epoch [3] Batch [3]: loss=0.91017\n", + "[INFO 2021-03-01 17:38:59,639] Epoch [3] Batch [4]: loss=0.85290\n", + "[INFO 2021-03-01 17:38:59,662] Epoch [3] Batch [5]: loss=0.81879\n", + "[INFO 2021-03-01 17:38:59,683] Epoch [3] Batch [6]: loss=0.79591\n", + "[INFO 2021-03-01 17:38:59,704] Epoch [3] Batch [7]: loss=0.77947\n", + "[INFO 2021-03-01 17:38:59,727] Epoch [3] Batch [8]: loss=0.76734\n", + "[INFO 2021-03-01 17:38:59,748] Epoch [3] Batch [9]: loss=0.75780\n", + "[INFO 2021-03-01 17:38:59,769] Epoch [3] Batch [10]: loss=0.75004\n", + "[INFO 2021-03-01 17:38:59,790] Epoch [3] Batch [11]: loss=0.74370\n", + "[INFO 2021-03-01 17:38:59,812] Epoch [3] Batch [12]: loss=0.73843\n", + "[INFO 2021-03-01 17:38:59,833] Epoch [3] Batch [13]: loss=0.73399\n", + "[INFO 2021-03-01 17:38:59,853] Epoch [3] Batch [14]: loss=0.73023\n", + "[INFO 2021-03-01 17:38:59,875] Epoch [3] Batch [15]: loss=0.72678\n", + "[INFO 2021-03-01 17:38:59,896] Epoch [3] Batch [16]: loss=0.72390\n", + "[INFO 2021-03-01 17:38:59,917] Epoch [3] Batch [17]: loss=0.72131\n", + "[INFO 2021-03-01 17:38:59,941] Epoch [3] Batch [18]: loss=0.71907\n", + "[INFO 2021-03-01 17:38:59,965] Epoch [3] Batch [19]: loss=0.71698\n", + "[INFO 2021-03-01 17:38:59,986] Epoch [3] Batch [20]: loss=0.71506\n", + "[INFO 2021-03-01 17:39:00,008] Epoch [3] Batch [21]: loss=0.71337\n", + "[INFO 2021-03-01 17:39:00,029] Epoch [3] Batch [22]: loss=0.71179\n", + "[INFO 2021-03-01 17:39:00,049] Epoch [3] Batch [23]: loss=0.71038\n", + "[INFO 2021-03-01 17:39:00,063] Epoch [3] Batch [24]: loss=0.70905\n", + "[INFO 2021-03-01 17:39:00,086] Epoch [4] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:00,107] Epoch [4] Batch [1]: loss=1.35415\n", + "[INFO 2021-03-01 17:39:00,128] Epoch [4] Batch [2]: loss=1.01525\n", + "[INFO 2021-03-01 17:39:00,149] Epoch [4] Batch [3]: loss=0.90241\n", + "[INFO 2021-03-01 17:39:00,170] Epoch [4] Batch [4]: loss=0.84585\n", + "[INFO 2021-03-01 17:39:00,191] Epoch [4] Batch [5]: loss=0.81217\n", + "[INFO 2021-03-01 17:39:00,212] Epoch [4] Batch [6]: loss=0.78947\n", + "[INFO 2021-03-01 17:39:00,233] Epoch [4] Batch [7]: loss=0.77324\n", + "[INFO 2021-03-01 17:39:00,255] Epoch [4] Batch [8]: loss=0.76119\n", + "[INFO 2021-03-01 17:39:00,278] Epoch [4] Batch [9]: loss=0.75163\n", + "[INFO 2021-03-01 17:39:00,300] Epoch [4] Batch [10]: loss=0.74402\n", + "[INFO 2021-03-01 17:39:00,322] Epoch [4] Batch [11]: loss=0.73773\n", + "[INFO 2021-03-01 17:39:00,344] Epoch [4] Batch [12]: loss=0.73245\n", + "[INFO 2021-03-01 17:39:00,367] Epoch [4] Batch [13]: loss=0.72797\n", + "[INFO 2021-03-01 17:39:00,388] Epoch [4] Batch [14]: loss=0.72407\n", + "[INFO 2021-03-01 17:39:00,411] Epoch [4] Batch [15]: loss=0.72082\n", + "[INFO 2021-03-01 17:39:00,432] Epoch [4] Batch [16]: loss=0.71783\n", + "[INFO 2021-03-01 17:39:00,455] Epoch [4] Batch [17]: loss=0.71533\n", + "[INFO 2021-03-01 17:39:00,478] Epoch [4] Batch [18]: loss=0.71306\n", + "[INFO 2021-03-01 17:39:00,501] Epoch [4] Batch [19]: loss=0.71098\n", + "[INFO 2021-03-01 17:39:00,522] Epoch [4] Batch [20]: loss=0.70911\n", + "[INFO 2021-03-01 17:39:00,545] Epoch [4] Batch [21]: loss=0.70745\n", + "[INFO 2021-03-01 17:39:00,567] Epoch [4] Batch [22]: loss=0.70583\n", + "[INFO 2021-03-01 17:39:00,588] Epoch [4] Batch [23]: loss=0.70437\n", + "[INFO 2021-03-01 17:39:00,601] Epoch [4] Batch [24]: loss=0.70301\n", + "[INFO 2021-03-01 17:39:00,622] Epoch [5] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:00,644] Epoch [5] Batch [1]: loss=1.34259\n", + "[INFO 2021-03-01 17:39:00,665] Epoch [5] Batch [2]: loss=1.00671\n", + "[INFO 2021-03-01 17:39:00,686] Epoch [5] Batch [3]: loss=0.89500\n", + "[INFO 2021-03-01 17:39:00,708] Epoch [5] Batch [4]: loss=0.83847\n", + "[INFO 2021-03-01 17:39:00,729] Epoch [5] Batch [5]: loss=0.80435\n", + "[INFO 2021-03-01 17:39:00,749] Epoch [5] Batch [6]: loss=0.78188\n", + "[INFO 2021-03-01 17:39:00,771] Epoch [5] Batch [7]: loss=0.76588\n", + "[INFO 2021-03-01 17:39:00,792] Epoch [5] Batch [8]: loss=0.75362\n", + "[INFO 2021-03-01 17:39:00,814] Epoch [5] Batch [9]: loss=0.74437\n", + "[INFO 2021-03-01 17:39:00,837] Epoch [5] Batch [10]: loss=0.73679\n", + "[INFO 2021-03-01 17:39:00,859] Epoch [5] Batch [11]: loss=0.73050\n", + "[INFO 2021-03-01 17:39:00,880] Epoch [5] Batch [12]: loss=0.72508\n", + "[INFO 2021-03-01 17:39:00,903] Epoch [5] Batch [13]: loss=0.72060\n", + "[INFO 2021-03-01 17:39:00,925] Epoch [5] Batch [14]: loss=0.71684\n", + "[INFO 2021-03-01 17:39:00,950] Epoch [5] Batch [15]: loss=0.71350\n", + "[INFO 2021-03-01 17:39:00,972] Epoch [5] Batch [16]: loss=0.71061\n", + "[INFO 2021-03-01 17:39:00,994] Epoch [5] Batch [17]: loss=0.70807\n", + "[INFO 2021-03-01 17:39:01,015] Epoch [5] Batch [18]: loss=0.70579\n", + "[INFO 2021-03-01 17:39:01,038] Epoch [5] Batch [19]: loss=0.70364\n", + "[INFO 2021-03-01 17:39:01,060] Epoch [5] Batch [20]: loss=0.70178\n", + "[INFO 2021-03-01 17:39:01,082] Epoch [5] Batch [21]: loss=0.70007\n", + "[INFO 2021-03-01 17:39:01,103] Epoch [5] Batch [22]: loss=0.69853\n", + "[INFO 2021-03-01 17:39:01,128] Epoch [5] Batch [23]: loss=0.69713\n", + "[INFO 2021-03-01 17:39:01,142] Epoch [5] Batch [24]: loss=0.69568\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[INFO 2021-03-01 17:39:01,165] Epoch [6] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:01,186] Epoch [6] Batch [1]: loss=1.32731\n", + "[INFO 2021-03-01 17:39:01,207] Epoch [6] Batch [2]: loss=0.99469\n", + "[INFO 2021-03-01 17:39:01,227] Epoch [6] Batch [3]: loss=0.88410\n", + "[INFO 2021-03-01 17:39:01,249] Epoch [6] Batch [4]: loss=0.82882\n", + "[INFO 2021-03-01 17:39:01,270] Epoch [6] Batch [5]: loss=0.79525\n", + "[INFO 2021-03-01 17:39:01,292] Epoch [6] Batch [6]: loss=0.77265\n", + "[INFO 2021-03-01 17:39:01,313] Epoch [6] Batch [7]: loss=0.75689\n", + "[INFO 2021-03-01 17:39:01,334] Epoch [6] Batch [8]: loss=0.74499\n", + "[INFO 2021-03-01 17:39:01,356] Epoch [6] Batch [9]: loss=0.73552\n", + "[INFO 2021-03-01 17:39:01,378] Epoch [6] Batch [10]: loss=0.72759\n", + "[INFO 2021-03-01 17:39:01,398] Epoch [6] Batch [11]: loss=0.72156\n", + "[INFO 2021-03-01 17:39:01,420] Epoch [6] Batch [12]: loss=0.71640\n", + "[INFO 2021-03-01 17:39:01,442] Epoch [6] Batch [13]: loss=0.71200\n", + "[INFO 2021-03-01 17:39:01,465] Epoch [6] Batch [14]: loss=0.70814\n", + "[INFO 2021-03-01 17:39:01,485] Epoch [6] Batch [15]: loss=0.70475\n", + "[INFO 2021-03-01 17:39:01,508] Epoch [6] Batch [16]: loss=0.70182\n", + "[INFO 2021-03-01 17:39:01,562] Epoch [6] Batch [17]: loss=0.69920\n", + "[INFO 2021-03-01 17:39:01,584] Epoch [6] Batch [18]: loss=0.69665\n", + "[INFO 2021-03-01 17:39:01,604] Epoch [6] Batch [19]: loss=0.69452\n", + "[INFO 2021-03-01 17:39:01,633] Epoch [6] Batch [20]: loss=0.69255\n", + "[INFO 2021-03-01 17:39:01,654] Epoch [6] Batch [21]: loss=0.69082\n", + "[INFO 2021-03-01 17:39:01,677] Epoch [6] Batch [22]: loss=0.68920\n", + "[INFO 2021-03-01 17:39:01,698] Epoch [6] Batch [23]: loss=0.68763\n", + "[INFO 2021-03-01 17:39:01,712] Epoch [6] Batch [24]: loss=0.68619\n", + "[INFO 2021-03-01 17:39:01,737] Epoch [7] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:01,760] Epoch [7] Batch [1]: loss=1.30147\n", + "[INFO 2021-03-01 17:39:01,781] Epoch [7] Batch [2]: loss=0.97596\n", + "[INFO 2021-03-01 17:39:01,802] Epoch [7] Batch [3]: loss=0.86841\n", + "[INFO 2021-03-01 17:39:01,823] Epoch [7] Batch [4]: loss=0.81361\n", + "[INFO 2021-03-01 17:39:01,844] Epoch [7] Batch [5]: loss=0.78113\n", + "[INFO 2021-03-01 17:39:01,865] Epoch [7] Batch [6]: loss=0.75962\n", + "[INFO 2021-03-01 17:39:01,885] Epoch [7] Batch [7]: loss=0.74389\n", + "[INFO 2021-03-01 17:39:01,906] Epoch [7] Batch [8]: loss=0.73186\n", + "[INFO 2021-03-01 17:39:01,929] Epoch [7] Batch [9]: loss=0.72272\n", + "[INFO 2021-03-01 17:39:01,950] Epoch [7] Batch [10]: loss=0.71526\n", + "[INFO 2021-03-01 17:39:01,971] Epoch [7] Batch [11]: loss=0.70912\n", + "[INFO 2021-03-01 17:39:01,992] Epoch [7] Batch [12]: loss=0.70406\n", + "[INFO 2021-03-01 17:39:02,014] Epoch [7] Batch [13]: loss=0.69967\n", + "[INFO 2021-03-01 17:39:02,034] Epoch [7] Batch [14]: loss=0.69580\n", + "[INFO 2021-03-01 17:39:02,058] Epoch [7] Batch [15]: loss=0.69247\n", + "[INFO 2021-03-01 17:39:02,079] Epoch [7] Batch [16]: loss=0.68941\n", + "[INFO 2021-03-01 17:39:02,101] Epoch [7] Batch [17]: loss=0.68691\n", + "[INFO 2021-03-01 17:39:02,124] Epoch [7] Batch [18]: loss=0.68440\n", + "[INFO 2021-03-01 17:39:02,146] Epoch [7] Batch [19]: loss=0.68225\n", + "[INFO 2021-03-01 17:39:02,167] Epoch [7] Batch [20]: loss=0.68027\n", + "[INFO 2021-03-01 17:39:02,189] Epoch [7] Batch [21]: loss=0.67852\n", + "[INFO 2021-03-01 17:39:02,210] Epoch [7] Batch [22]: loss=0.67680\n", + "[INFO 2021-03-01 17:39:02,231] Epoch [7] Batch [23]: loss=0.67525\n", + "[INFO 2021-03-01 17:39:02,245] Epoch [7] Batch [24]: loss=0.67364\n", + "[INFO 2021-03-01 17:39:02,267] Epoch [8] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:02,288] Epoch [8] Batch [1]: loss=1.27866\n", + "[INFO 2021-03-01 17:39:02,309] Epoch [8] Batch [2]: loss=0.95875\n", + "[INFO 2021-03-01 17:39:02,331] Epoch [8] Batch [3]: loss=0.85147\n", + "[INFO 2021-03-01 17:39:02,352] Epoch [8] Batch [4]: loss=0.79788\n", + "[INFO 2021-03-01 17:39:02,372] Epoch [8] Batch [5]: loss=0.76482\n", + "[INFO 2021-03-01 17:39:02,393] Epoch [8] Batch [6]: loss=0.74289\n", + "[INFO 2021-03-01 17:39:02,415] Epoch [8] Batch [7]: loss=0.72653\n", + "[INFO 2021-03-01 17:39:02,436] Epoch [8] Batch [8]: loss=0.71534\n", + "[INFO 2021-03-01 17:39:02,457] Epoch [8] Batch [9]: loss=0.70576\n", + "[INFO 2021-03-01 17:39:02,478] Epoch [8] Batch [10]: loss=0.69839\n", + "[INFO 2021-03-01 17:39:02,498] Epoch [8] Batch [11]: loss=0.69218\n", + "[INFO 2021-03-01 17:39:02,519] Epoch [8] Batch [12]: loss=0.68720\n", + "[INFO 2021-03-01 17:39:02,540] Epoch [8] Batch [13]: loss=0.68280\n", + "[INFO 2021-03-01 17:39:02,561] Epoch [8] Batch [14]: loss=0.67918\n", + "[INFO 2021-03-01 17:39:02,582] Epoch [8] Batch [15]: loss=0.67622\n", + "[INFO 2021-03-01 17:39:02,603] Epoch [8] Batch [16]: loss=0.67336\n", + "[INFO 2021-03-01 17:39:02,625] Epoch [8] Batch [17]: loss=0.67071\n", + "[INFO 2021-03-01 17:39:02,646] Epoch [8] Batch [18]: loss=0.66827\n", + "[INFO 2021-03-01 17:39:02,667] Epoch [8] Batch [19]: loss=0.66600\n", + "[INFO 2021-03-01 17:39:02,689] Epoch [8] Batch [20]: loss=0.66409\n", + "[INFO 2021-03-01 17:39:02,711] Epoch [8] Batch [21]: loss=0.66243\n", + "[INFO 2021-03-01 17:39:02,732] Epoch [8] Batch [22]: loss=0.66077\n", + "[INFO 2021-03-01 17:39:02,753] Epoch [8] Batch [23]: loss=0.65927\n", + "[INFO 2021-03-01 17:39:02,767] Epoch [8] Batch [24]: loss=0.65789\n", + "[INFO 2021-03-01 17:39:02,789] Epoch [9] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:02,810] Epoch [9] Batch [1]: loss=1.23720\n", + "[INFO 2021-03-01 17:39:02,832] Epoch [9] Batch [2]: loss=0.92866\n", + "[INFO 2021-03-01 17:39:02,855] Epoch [9] Batch [3]: loss=0.82470\n", + "[INFO 2021-03-01 17:39:02,876] Epoch [9] Batch [4]: loss=0.77277\n", + "[INFO 2021-03-01 17:39:02,898] Epoch [9] Batch [5]: loss=0.74245\n", + "[INFO 2021-03-01 17:39:02,919] Epoch [9] Batch [6]: loss=0.72158\n", + "[INFO 2021-03-01 17:39:02,941] Epoch [9] Batch [7]: loss=0.70738\n", + "[INFO 2021-03-01 17:39:02,963] Epoch [9] Batch [8]: loss=0.69623\n", + "[INFO 2021-03-01 17:39:02,986] Epoch [9] Batch [9]: loss=0.68729\n", + "[INFO 2021-03-01 17:39:03,006] Epoch [9] Batch [10]: loss=0.67998\n", + "[INFO 2021-03-01 17:39:03,029] Epoch [9] Batch [11]: loss=0.67417\n", + "[INFO 2021-03-01 17:39:03,050] Epoch [9] Batch [12]: loss=0.66928\n", + "[INFO 2021-03-01 17:39:03,071] Epoch [9] Batch [13]: loss=0.66498\n", + "[INFO 2021-03-01 17:39:03,093] Epoch [9] Batch [14]: loss=0.66116\n", + "[INFO 2021-03-01 17:39:03,114] Epoch [9] Batch [15]: loss=0.65791\n", + "[INFO 2021-03-01 17:39:03,134] Epoch [9] Batch [16]: loss=0.65492\n", + "[INFO 2021-03-01 17:39:03,156] Epoch [9] Batch [17]: loss=0.65273\n", + "[INFO 2021-03-01 17:39:03,178] Epoch [9] Batch [18]: loss=0.65027\n", + "[INFO 2021-03-01 17:39:03,201] Epoch [9] Batch [19]: loss=0.64776\n", + "[INFO 2021-03-01 17:39:03,222] Epoch [9] Batch [20]: loss=0.64532\n", + "[INFO 2021-03-01 17:39:03,243] Epoch [9] Batch [21]: loss=0.64357\n", + "[INFO 2021-03-01 17:39:03,264] Epoch [9] Batch [22]: loss=0.64178\n", + "[INFO 2021-03-01 17:39:03,288] Epoch [9] Batch [23]: loss=0.64034\n", + "[INFO 2021-03-01 17:39:03,303] Epoch [9] Batch [24]: loss=0.63925\n", + "[INFO 2021-03-01 17:39:03,325] Epoch [10] Batch [0]: loss=inf\n", + "[INFO 2021-03-01 17:39:03,346] Epoch [10] Batch [1]: loss=1.20359\n", + "[INFO 2021-03-01 17:39:03,367] Epoch [10] Batch [2]: loss=0.90176\n", + "[INFO 2021-03-01 17:39:03,391] Epoch [10] Batch [3]: loss=0.80220\n", + "[INFO 2021-03-01 17:39:03,411] Epoch [10] Batch [4]: loss=0.75159\n", + "[INFO 2021-03-01 17:39:03,432] Epoch [10] Batch [5]: loss=0.72171\n", + "[INFO 2021-03-01 17:39:03,454] Epoch [10] Batch [6]: loss=0.69995\n", + "[INFO 2021-03-01 17:39:03,475] Epoch [10] Batch [7]: loss=0.68538\n", + "[INFO 2021-03-01 17:39:03,496] Epoch [10] Batch [8]: loss=0.67389\n", + "[INFO 2021-03-01 17:39:03,517] Epoch [10] Batch [9]: loss=0.66552\n", + "[INFO 2021-03-01 17:39:03,538] Epoch [10] Batch [10]: loss=0.65826\n", + "[INFO 2021-03-01 17:39:03,561] Epoch [10] Batch [11]: loss=0.65266\n", + "[INFO 2021-03-01 17:39:03,587] Epoch [10] Batch [12]: loss=0.64794\n", + "[INFO 2021-03-01 17:39:03,610] Epoch [10] Batch [13]: loss=0.64335\n", + "[INFO 2021-03-01 17:39:03,631] Epoch [10] Batch [14]: loss=0.63941\n", + "[INFO 2021-03-01 17:39:03,652] Epoch [10] Batch [15]: loss=0.63602\n", + "[INFO 2021-03-01 17:39:03,708] Epoch [10] Batch [16]: loss=0.63353\n", + "[INFO 2021-03-01 17:39:03,730] Epoch [10] Batch [17]: loss=0.63105\n", + "[INFO 2021-03-01 17:39:03,752] Epoch [10] Batch [18]: loss=0.62908\n", + "[INFO 2021-03-01 17:39:03,773] Epoch [10] Batch [19]: loss=0.62696\n", + "[INFO 2021-03-01 17:39:03,794] Epoch [10] Batch [20]: loss=0.62514\n", + "[INFO 2021-03-01 17:39:03,816] Epoch [10] Batch [21]: loss=0.62364\n", + "[INFO 2021-03-01 17:39:03,838] Epoch [10] Batch [22]: loss=0.62210\n", + "[INFO 2021-03-01 17:39:03,860] Epoch [10] Batch [23]: loss=0.62031\n", + "[INFO 2021-03-01 17:39:03,874] Epoch [10] Batch [24]: loss=0.61861\n" + ] + } + ], + "source": [ + "# define model here\n", + "model = CAT.model.IRTModel(**config)\n", + "# train model\n", + "model.init_model(train_data)\n", + "model.train(train_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# save model\n", + "model.adaptest_save('../ckpt/checkpoint_mirt.pt')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}