{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import CAT\n",
    "import json\n",
    "import torch\n",
    "import logging\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setuplogger():\n",
    "    root = logging.getLogger()\n",
    "    root.setLevel(logging.INFO)\n",
    "    handler = logging.StreamHandler(sys.stdout)\n",
    "    handler.setLevel(logging.INFO)\n",
    "    formatter = logging.Formatter(\"[%(levelname)s %(asctime)s] %(message)s\")\n",
    "    handler.setFormatter(formatter)\n",
    "    root.addHandler(handler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setuplogger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tensorboard\n",
    "log_dir = f\"../logs/{datetime.datetime.now().strftime('%Y-%m-%d-%H:%M')}/\"\n",
    "log_dir = f\"../logs/\"\n",
    "print(log_dir)\n",
    "writer = SummaryWriter(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose dataset here\n",
    "import CAT.strategy\n",
    "\n",
    "\n",
    "dataset = 'assistment'\n",
    "# modify config here\n",
    "config = {\n",
    "    'learning_rate': 0.0025,\n",
    "    'batch_size': 2048,\n",
    "    'num_epochs': 8,\n",
    "    'num_dim': 1, # for IRT or MIRT\n",
    "    'device': 'cpu',\n",
    "    # for NeuralCD\n",
    "    'prednet_len1': 128,\n",
    "    'prednet_len2': 64,\n",
    "    # for BOBCAT\n",
    "    'policy':'notbobcat',\n",
    "    'betas': (0.9, 0.999),\n",
    "    'policy_path': 'policy.pt',\n",
    "    # for NCAT\n",
    "    'THRESHOLD' :300,\n",
    "    'start':0,\n",
    "    'end':3000\n",
    "    \n",
    "}\n",
    "# fixed test length\n",
    "test_length = 5\n",
    "# choose strategies here\n",
    "#strategies = [CAT.strategy.RandomStrategy(), CAT.strategy.MFIStrategy(), CAT.strategy.KLIStrategy()]\n",
    "strategies = [CAT.strategy.NCATs()]\n",
    "# modify checkpoint path here\n",
    "ckpt_path = '../ckpt/irt.pt'\n",
    "bobcat_policy_path =config['policy_path']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read datasets\n",
    "test_triplets = pd.read_csv(f'../data/{dataset}/test_triples.csv', encoding='utf-8').to_records(index=False)\n",
    "concept_map = json.load(open(f'../data/{dataset}/concept_map.json', 'r'))\n",
    "concept_map = {int(k):v for k,v in concept_map.items()}\n",
    "metadata = json.load(open(f'../data/{dataset}/metadata.json', 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = CAT.dataset.AdapTestDataset(test_triplets, concept_map,\n",
    "                                        metadata['num_test_students'], \n",
    "                                        metadata['num_questions'], \n",
    "                                        metadata['num_concepts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for strategy in strategies:\n",
    "    avg =[]\n",
    "    model = CAT.model.IRTModel(**config)\n",
    "    #model = CAT.model.NCDModel(**config)\n",
    "    model.init_model(test_data)\n",
    "    model.adaptest_load(ckpt_path)\n",
    "    test_data.reset()\n",
    "    print(strategy.name)\n",
    "    if strategy.name == 'NCAT':\n",
    "        selected_questions = strategy.adaptest_select(test_data,concept_map,config,test_length)\n",
    "        for it in range(test_length):\n",
    "            for student, questions in selected_questions.items():\n",
    "                test_data.apply_selection(student, questions[it])  \n",
    "            model.adaptest_update(test_data)\n",
    "            results = model.evaluate(test_data)\n",
    "        # log results\n",
    "            logging.info(f'Iteration {it}')\n",
    "            for name, value in results.items():\n",
    "                logging.info(f'{name}:{value}')\n",
    "        continue\n",
    "    if strategy.name == 'BOBCAT':\n",
    "        real = {}\n",
    "        real_data = test_data.data\n",
    "        for sid in real_data:\n",
    "            question_ids = list(real_data[sid].keys())\n",
    "            real[sid]={}\n",
    "            tmp={}\n",
    "            for qid in question_ids:\n",
    "                tmp[qid]=real_data[sid][qid]\n",
    "            real[sid]=tmp\n",
    "    logging.info('-----------')\n",
    "    logging.info(f'start adaptive testing with {strategy.name} strategy')\n",
    "    logging.info(f'Iteration 0')\n",
    "    # evaluate models\n",
    "    results = model.evaluate(test_data)\n",
    "    for name, value in results.items():\n",
    "        logging.info(f'{name}:{value}')\n",
    "    S_sel ={}\n",
    "    for sid in range(test_data.num_students):\n",
    "        key = sid\n",
    "        S_sel[key] = []\n",
    "    selected_questions={}\n",
    "    for it in range(1, test_length + 1):\n",
    "        logging.info(f'Iteration {it}')\n",
    "        # select question\n",
    "        if strategy.name == 'BOBCAT':\n",
    "            selected_questions = strategy.adaptest_select(model, test_data,S_sel)\n",
    "            for sid in range(test_data.num_students):\n",
    "                tmp = {}\n",
    "                tmp[selected_questions[sid]] = real[sid][selected_questions[sid]]\n",
    "                S_sel[sid].append(tmp)\n",
    "        elif it == 1 and strategy.name == 'BECAT Strategy':\n",
    "            for sid in range(test_data.num_students):\n",
    "                untested_questions = np.array(list(test_data.untested[sid]))\n",
    "                random_index = random.randint(0, len(untested_questions)-1)\n",
    "                selected_questions[sid] = untested_questions[random_index]\n",
    "                S_sel[sid].append(untested_questions[random_index])\n",
    "        elif strategy.name == 'BECAT Strategy':    \n",
    "            selected_questions = strategy.adaptest_select(model, test_data,S_sel)\n",
    "            for sid in range(test_data.num_students):\n",
    "                S_sel[sid].append(selected_questions[sid])\n",
    "        else:\n",
    "            selected_questions = strategy.adaptest_select(model, test_data)\n",
    "        for student, question in selected_questions.items():\n",
    "            test_data.apply_selection(student, question)       \n",
    "        \n",
    "        # update models\n",
    "        model.adaptest_update(test_data)\n",
    "        # evaluate models\n",
    "        results = model.evaluate(test_data)\n",
    "        # log results\n",
    "        for name, value in results.items():\n",
    "            logging.info(f'{name}:{value}')\n",
    "            writer.add_scalars(name, {strategy.name: value}, it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}