Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Usage:\n",
    "#     classify_xvsy_logreg.py [options] <first> <second>\n",
    "#     classify_xvsy_logreg.py [options] --all-vs <second>\n",
    "\n",
    "# Options:\n",
    "#     -j --json=<JSONFILE>  Filename of the json file [default: unified-dataset.jsonl]\n",
    "#     -a --all-vs=<dataset> Dataset name of the testing data\n",
    "#     -d --debug            Use a small word list and a fast classifier\n",
    "#     -o --output=<OUTPUT>  Output folder [default: .]\n",
    "#     -m --force-multi      Force using multi-label classification\n",
    "#     -k --keep-last        Quit immediately if results file found\n",
    "\n",
    "import regex as re\n",
    "import sys\n",
    "import os\n",
    "import json\n",
    "import random\n",
    "import math\n",
    "import operator as op\n",
    "import docopt\n",
    "import numpy as np\n",
    "import os.path\n",
    "from os import path\n",
    "from tqdm import tqdm\n",
    "\n",
    "from collections import Counter, defaultdict, namedtuple\n",
    "\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "from sklearn.metrics import classification_report\n",
    "from scipy.spatial import distance\n",
    "import joblib\n",
    "import shutil\n",
    "\n",
    "np.random.seed(0)\n",
    "random.seed(0)\n",
    "\n",
    "Report = namedtuple(\n",
    "    \"Report\", [\"precision\", \"recall\", \"accuracy\", \"f1\", \"tp\", \"tn\", \"fp\", \"fn\"]\n",
    ")\n",
    "\n",
    "PATTERN_TOKENS = re.compile(r\"[a-z]+\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cheatydiv(x, y):\n",
    "    return math.nan if y == 0 else x / y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_labels(train, test, operation=op.and_, mode=\"multi\"):\n",
    "    \"\"\"Return a list of the emotional intersection of two sources.\"\"\"\n",
    "    emotions = set()\n",
    "    if mode == \"single\":\n",
    "        emotions.add(\"noemo\")\n",
    "    train_emotions = set(\n",
    "        emotion\n",
    "        for data in train\n",
    "        for emotion in data[\"emotions\"]\n",
    "        if data[\"emotions\"][emotion] is not None\n",
    "    )\n",
    "    # print(train_emotions)\n",
    "    test_emotions = set(\n",
    "        emotion\n",
    "        for emotion in test[0][\"emotions\"]\n",
    "        if test[0][\"emotions\"][emotion] is not None\n",
    "    )\n",
    "    # print(test_emotions)\n",
    "    return list(emotions | operation(train_emotions, test_emotions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_emotion(emovals, labels, emotions, mode=\"multi\"):\n",
    "#     print(\"get emotion mode \", mode)\n",
    "#     print(\"emovals \",emovals)\n",
    "#     print(\"labels \",labels)\n",
    "#     print(\"emotions \",emotions)\n",
    "    \n",
    "    if mode == \"single\":\n",
    "        truthy = len(list(filter(bool, emovals.values())))\n",
    "        if truthy == 1:\n",
    "            emotion = [v for v in emovals if emovals[v]][0]\n",
    "        elif truthy == 0:\n",
    "            emotion = \"noemo\"\n",
    "        else:\n",
    "            # emotion = sorted(\n",
    "            #     ((k, v) for k, v in emovals.items() if v),\n",
    "            #     key=lambda x: x[1],\n",
    "            #     reverse=True,\n",
    "            # )[0][0]\n",
    "            raise ValueError(\"Dataset marked as 'single' contains multiple emotions\")\n",
    "        return emotions.get(emotion, emotions.get(\"noemo\"))\n",
    "    else:\n",
    "        el = [int((emovals[label] or 0) > 0.1) for label in labels]\n",
    "        return np.array(el)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vector(text, wordlist):\n",
    "    tokens = set(tokenize(text))\n",
    "    return [1 if word in tokens else 0 for word in wordlist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_arrays(train, test, words, labels, mode=\"multi\", all_vs=False):\n",
    "    emotions = {label: x for x, label in enumerate(labels)}\n",
    "    print(\"emotions in make_arrays: \", emotions)\n",
    "    train_x, train_y, test_x, test_y = [], [], [], []\n",
    "    \n",
    "    print(\"train super raw: \", sys.getsizeof(train)/1000000)\n",
    "#     print(\"train:\", train)\n",
    "\n",
    "    for data in tqdm(train):\n",
    "        # Discard examples where we don't have all selected emotions\n",
    "        if (mode == \"single\" or all_vs or all(data[\"emotions\"][emo] is not None for emo in labels)):\n",
    "            train_y.append(get_emotion(data[\"emotions\"], labels, emotions, mode))\n",
    "            train_x.append(get_vector(data[\"text\"], words))\n",
    "    for data in tqdm(test):\n",
    "        test_y.append(get_emotion(data[\"emotions\"], labels, emotions, mode))\n",
    "        test_x.append(get_vector(data[\"text\"], words))\n",
    "#     joblib.dump(train_x, train_xFileName)\n",
    "#     joblib.dump(train_y, train_yFileName)\n",
    "#     joblib.dump(test_x, test_xFileName)\n",
    "#     joblib.dump(test_y, test_yFileName)\n",
    "#     print(\"saved train and test data\")\n",
    "#     print(\"train_x item:\", train_x[0])\n",
    "    print(\"train_x length \", len(train_x))\n",
    "    print(\"train_x dimension of element \", len(train_x[0]))\n",
    "    train_xSize = sys.getsizeof(train_x)/1000000\n",
    "    train_ySize = sys.getsizeof(train_y)/1000000\n",
    "    train_xLength = len(train_x)\n",
    "    train_yLength = len(train_y)\n",
    "    print(\"train_x (text) size RAW:\", train_xSize,\"megabytes\")\n",
    "    print(\"train_y (labels) size RAW:\", train_ySize,\"megabytes\")\n",
    "    test_xSize = sys.getsizeof(test_x)/1000000\n",
    "    test_ySize = sys.getsizeof(test_y)/1000000\n",
    "    test_xLength = len(test_x)\n",
    "    test_yLength = len(test_y)\n",
    "    print(\"test_x (text) size RAW:\", test_xSize,\"megabytes\")\n",
    "    print(\"test_y (labels) size RAW:\", test_ySize,\"megabytes\")\n",
    "#     if(saveProcessedArrays):\n",
    "#         print('saved train_x as', train_xFileName)\n",
    "#         print('saved train_y as', train_yFileName)\n",
    "#         print('saved test_x as', test_xFileName)\n",
    "#         print('saved test_y as', test_yFileName)\n",
    "\n",
    "    train_x = np.array(train_x)\n",
    "    train_y = np.array(train_y)\n",
    "    test_x = np.array(test_x)\n",
    "    test_y = np.array(test_y)\n",
    "    train_xNPSize = (train_x.nbytes)/1000000\n",
    "    train_yNPSize = (train_y.nbytes)/1000000\n",
    "    test_xNPSize = (test_x.nbytes)/1000000\n",
    "    test_yNPSize = (test_y.nbytes)/1000000\n",
    "    \n",
    "#     train_xFileName = first + \"_\" + second + \"train_x\" +\".pkl\"\n",
    "#     train_yFileName = first + \"_\" + second + \"train_y\" +\".pkl\"\n",
    "#     test_xFileName = first + \"_\" + second + \"test_x\" +\".pkl\"\n",
    "#     test_yFileName = first + \"_\" + second + \"test_y\" +\".pkl\"\n",
    "#     if(path.exists(train_xFileName) \n",
    "#        and path.exists(train_yFileName)\n",
    "#        and path.exists(test_xFileName)\n",
    "#        and path.exists(test_yFileName)): #if processed arrays exist, load them in\n",
    "#         print(\"successfully located paths for train and test data\")\n",
    "#         train_x1 = joblib.load(train_xFileName)\n",
    "#         train_y1 = joblib.load(train_yFileName)\n",
    "#         test_x1 = joblib.load(test_xFileName)\n",
    "#         test_y1 = joblib.load(test_yFileName)\n",
    "#         print(\"loaded train and test data\")\n",
    "#         train_x1 = np.array(train_x1)\n",
    "#         train_y1 = np.array(train_y1)\n",
    "#         test_x1 = np.array(test_x1)\n",
    "#         test_y1 = np.array(test_y1)\n",
    "#         train_x1NPSize = (train_x1.nbytes)/1000000\n",
    "#         train_y1NPSize = (train_y1.nbytes)/1000000\n",
    "#         test_x1NPSize = (test_x1.nbytes)/1000000\n",
    "#         test_y1NPSize = (test_y1.nbytes)/1000000\n",
    "#         print(\"train_x Size stays the same loaded\", train_xSize == train_x1NPSize)\n",
    "#         print(\"train_y Size stays the same loaded\", train_ySize == train_y1NPSize)\n",
    "#         print(\"test_x Size stays the same loaded\", test_xSize == test_xNPSize)\n",
    "#         print(\"test_y Size stays the same loaded\", test_ySize == test_y1NPSize)\n",
    "#         print(\"train_xNPSize (text) size loaded:\", train_x1NPSize,\"megabytes\")\n",
    "#         print(\"train_yNPSize (labels) size loaded:\", train_y1NPSize,\"megabytes\")\n",
    "#         print(\"test_xNPSize (text) size loaded:\", test_x1NPSize,\"megabytes\")\n",
    "#         print(\"test_yNPSize (labels) size loaded:\", test_y1NPSize,\"megabytes\")\n",
    "    print(\"saved test_y\")\n",
    "    print(\"train_x Size stays the same\", train_xSize == train_xNPSize)\n",
    "    print(\"train_y Size stays the same\", train_ySize == train_yNPSize)\n",
    "    print(\"test_x Size stays the same\", test_xSize == test_xNPSize)\n",
    "    print(\"test_y Size stays the same\", test_ySize == test_yNPSize)\n",
    "    print(\"train_xNPSize (text) size:\", train_xNPSize,\"megabytes\")\n",
    "    print(\"train_yNPSize (labels) size:\", train_yNPSize,\"megabytes\")\n",
    "    print(\"test_xNPSize (text) size:\", test_xNPSize,\"megabytes\")\n",
    "    print(\"test_yNPSize (labels) size:\", test_yNPSize,\"megabytes\")\n",
    "    print(\"train_xNP length \", len(train_x))\n",
    "    print(\"train_xNP dimension of element \", train_x.ndim)\n",
    "    print(\"train_xNP size \", train_x.size)\n",
    "    sizes = train_xNPSize, train_yNPSize, test_xNPSize, test_yNPSize\n",
    "    return train_x, train_y, test_x, test_y, sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filtered_texts(filename, source):\n",
    "    with open(filename) as f:\n",
    "        for line in f:\n",
    "            data = json.loads(line)\n",
    "            if data[\"source\"] == source:\n",
    "                yield data[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classification_report_own_single(test_y, predict_y, labels):\n",
    "    reports = {}\n",
    "    num2emo = {i: label for i, label in enumerate(labels)}\n",
    "    decisions = defaultdict(Counter)\n",
    "    for t, p in zip(test_y, predict_y):\n",
    "        decisions[t][p] += 1\n",
    "    for label in decisions:\n",
    "        tp = decisions[label][label]\n",
    "        fp = sum(decisions[x][label] for x in decisions if x != label)\n",
    "        tn = sum(\n",
    "            decisions[x][y]\n",
    "            for x in decisions\n",
    "            for y in decisions[x]\n",
    "            if x != label and y != label\n",
    "        )\n",
    "        fn = sum(decisions[label][y] for y in decisions[label] if y != label)\n",
    "        precision = tp / (tp + fp) if tp + fp else math.nan\n",
    "        recall = tp / (tp + fn) if tp + fn else math.nan\n",
    "        f1 = 2 * cheatydiv((precision * recall), (precision + recall))\n",
    "        accuracy = (tp + tn) / (tp + tn + fp + fn)\n",
    "        reports[num2emo[label]] = Report(\n",
    "            precision, recall, accuracy, f1, tp, tn, fp, fn\n",
    "        )\n",
    "    return reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classification_report_own_multi(test_y, predict_y, labels):\n",
    "    reports = {}\n",
    "    num2emo = {i: label for i, label in enumerate(labels)}\n",
    "    emo2num = {label: i for i, label in enumerate(labels)}\n",
    "    decisions = defaultdict(Counter)\n",
    "    for label in labels:\n",
    "        tp = fp = tn = fn = 0\n",
    "        for t, p in zip(test_y, predict_y):\n",
    "            # decisions[t][p] += 1\n",
    "            tp += bool(t[emo2num[label]] and p[emo2num[label]])\n",
    "            fp += bool(p[emo2num[label]] and not t[emo2num[label]])\n",
    "            fn += bool(t[emo2num[label]] and not p[emo2num[label]])\n",
    "            tn += bool(not t[emo2num[label]] and not p[emo2num[label]])\n",
    "        precision = tp / (tp + fp) if tp + fp else math.nan\n",
    "        recall = tp / (tp + fn) if tp + fn else math.nan\n",
    "        f1 = 2 * cheatydiv((precision * recall), (precision + recall))\n",
    "        accuracy = (tp + tn) / (tp + tn + fp + fn)\n",
    "        reports[label] = Report(precision, recall, accuracy, f1, tp, tn, fp, fn)\n",
    "    return reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    return re.findall(r\"\\p{L}+\", text.lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is bad. memory error for all_vs (too many words...)\n",
    "def get_wordlist(dataset):\n",
    "    \"\"\"Get a bag of words from a dataset.\"\"\"\n",
    "    bag = set()\n",
    "    for data in dataset:\n",
    "        bag.update({token for token in tokenize(data[\"text\"])})\n",
    "    return list(bag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ask roman what would be a good vocab here?\n",
    "def get_wordlist_debug(dataset):\n",
    "    \"\"\"Get a bag of words from a dataset.\"\"\"\n",
    "    bag = Counter()\n",
    "    for data in dataset:\n",
    "        bag.update({token for token in tokenize(data[\"text\"])})\n",
    "    return list(map(op.itemgetter(0), bag.most_common(5000)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hacky_train_test_split(training, train_size=0.8, first=None, second=None):\n",
    "    tra, tes = [], []\n",
    "    for example in training:\n",
    "        if example.get(\"split\") == \"train\" or example[\"source\"] != second:\n",
    "            tra.append(example)\n",
    "        elif example.get(\"split\") == \"test\":\n",
    "            tes.append(example)\n",
    "        else:\n",
    "            # don't try this at home\n",
    "            [tes, tra][random.random()<train_size].append(example)\n",
    "    return tra, tes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test(jsonfile, train, test):\n",
    "    print(\"get_train_test param:\")\n",
    "    print(\"json \", jsonfile)\n",
    "    print(\"train \", train)\n",
    "    print(\"test \", test)\n",
    "#     same = test in train.split(\",\") #used if train and test corpus are same\n",
    "    training, testing = [], []\n",
    "    count1 = 0\n",
    "    count2 = 0\n",
    "    count3 = 0\n",
    "    count4 = 0\n",
    "    with open(jsonfile) as f:\n",
    "        for line in f:\n",
    "            data = json.loads(line)\n",
    "            if(data[\"source\"] == test):\n",
    "                count1 += 1\n",
    "            if(data[\"source\"] != test):\n",
    "                count2 += 1\n",
    "            if(train == None and data[\"source\"] != test):\n",
    "                count3 += 1\n",
    "                training.append(data)\n",
    "            elif data[\"source\"] == test:\n",
    "                count4 += 1\n",
    "                testing.append(data)\n",
    "            elif(data[\"source\"] in train.split(\",\")):\n",
    "                count3 += 1\n",
    "                training.append(data)\n",
    "    print(\"there were \", count1, \" entries that were in test and \", count2, \"that were not in test\",\n",
    "          \"and \", count3, \" that were in train\")\n",
    "    print(\"test was appended \", count4, \" times\")\n",
    "#     if same:\n",
    "#         training, testing = hacky_train_test_split(training, train_size=0.8, first=train, second=test)\n",
    "    return training, testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_clf_mode(train, test):\n",
    "    \"\"\" Detect whether we are in single-label to single-label mode or not. \"\"\"\n",
    "    first = \"single\"\n",
    "    for example in train:\n",
    "        if example.get(\"labeled\", \"multi\") == \"multi\":\n",
    "            first = \"multi\"\n",
    "    print(first)\n",
    "    for example in test:\n",
    "        if example.get(\"labeled\", \"multi\") == \"multi\":\n",
    "            return first, \"multi\"\n",
    "    print(\"oof\")\n",
    "    return first, \"single\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_results(test_y, predict_y, labels, test, first, second, output, mode):\n",
    "    print(\"analyse_results\")\n",
    "    prefix = f\"{first}_vs_{second}_{mode}\"\n",
    "    fprefix = output + \"/\" + prefix\n",
    "    with open(fprefix + \".txt\", \"w\", encoding=\"utf-8\") as f, open(fprefix + \".json\", \"w\") as g:\n",
    "        print(\"hello\")\n",
    "        # print(confusion_matrix(test_y, predict_y), file=f)\n",
    "        prec, reca, f1, supp = precision_recall_fscore_support(\n",
    "            test_y, predict_y, pos_label=None, average=\"micro\"\n",
    "        )\n",
    "        accuracy = accuracy_score(test_y, predict_y)\n",
    "        scoreNameArray = [(prec, \"Precision\"),(reca, \"Recall\"),(f1, \"F1-score\"),(accuracy, \"Accuracy\")]\n",
    "        for score, name in scoreNameArray:\n",
    "            print(name, score, sep=\"\\t\", file=f)\n",
    "            print(name, score, sep=\"\\t\")\n",
    "\n",
    "        # print(\"real:\", Counter(test_y), file=f)\n",
    "        # print(\"predicted:\", Counter(predict_y), file=f)\n",
    "\n",
    "        print(test_y[:10], predict_y[:10], file=f)\n",
    "        emotions = {i: label for i, label in enumerate(labels)}\n",
    "        for text, real, predicted, _ in zip(test, test_y, predict_y, range(20)):\n",
    "            if mode == \"multi\" and np.array_equal(real, predicted):\n",
    "                continue\n",
    "            elif mode == \"single\" and real == predicted:\n",
    "                continue\n",
    "            print(text, \"=> predicted:\", predicted, \", truth:\", real, file=f)\n",
    "        if mode == \"multi\":\n",
    "            results = classification_report_own_multi(test_y, predict_y, labels)\n",
    "        elif mode == \"single\":\n",
    "            results = classification_report_own_single(test_y, predict_y, labels)\n",
    "        json.dump(\n",
    "            {\n",
    "                \"precision\": prec,\n",
    "                \"recall\": reca,\n",
    "                \"f1\": f1,\n",
    "                \"accuracy\": accuracy,\n",
    "                \"name\": prefix,\n",
    "                **{\n",
    "                    (emotion + \"_\" + metric): getattr(results[emotion], metric)\n",
    "                    for emotion in results\n",
    "                    for metric in Report._fields\n",
    "                },\n",
    "            },\n",
    "            g,\n",
    "        )\n",
    "        g.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPowerset(s):\n",
    "    x = len(s)\n",
    "    masks = [1 << i for i in range(x)]\n",
    "    for i in range(1 << x):\n",
    "        yield [ss for mask, ss in zip(masks, s) if i & mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "def getPermutations(s):\n",
    "    subsets = set()\n",
    "    for L in range(2, 3):\n",
    "        for subset in itertools.permutations(s, L):\n",
    "#             print(subset)\n",
    "            subsets.add(subset)\n",
    "    return subsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getHardDriveSpaceLeft():\n",
    "    total, used, free = shutil.disk_usage(\"/\")\n",
    "    total = (total // (2**30))\n",
    "    used = (used // (2**30))\n",
    "    free = (free // (2**30))\n",
    "    print(\"Total: %d GiB\" % total)\n",
    "    print(\"Used: %d GiB\" % used)\n",
    "    print(\"Free: %d GiB\" % free)\n",
    "    return total, used, free"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "permutations length:  110\n",
      "('tec', 'tales-emotion', 'tweets', 'tales')\n",
      "('tec', 'ssec', 'tweets', 'tweets')\n",
      "('tec', 'isear', 'tweets', 'descriptions')\n",
      "('tec', 'grounded_emotions', 'tweets', 'tweets')\n",
      "('tec', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "('tec', 'emoint', 'tweets', 'tweets')\n",
      "('tec', 'emobank', 'tweets', 'headlines')\n",
      "('tec', 'dailydialog', 'tweets', 'conversations')\n",
      "('tec', 'crowdflower', 'tweets', 'tweets')\n",
      "('tec', 'affectivetext', 'tweets', 'headlines')\n",
      "('ssec', 'tec', 'tweets', 'tweets')\n",
      "('ssec', 'tales-emotion', 'tweets', 'tales')\n",
      "('ssec', 'isear', 'tweets', 'descriptions')\n",
      "('ssec', 'grounded_emotions', 'tweets', 'tweets')\n",
      "('ssec', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "('ssec', 'emoint', 'tweets', 'tweets')\n",
      "('ssec', 'emobank', 'tweets', 'headlines')\n",
      "('ssec', 'dailydialog', 'tweets', 'conversations')\n",
      "('ssec', 'crowdflower', 'tweets', 'tweets')\n",
      "('ssec', 'affectivetext', 'tweets', 'headlines')\n",
      "('grounded_emotions', 'tec', 'tweets', 'tweets')\n",
      "('grounded_emotions', 'tales-emotion', 'tweets', 'tales')\n",
      "('grounded_emotions', 'ssec', 'tweets', 'tweets')\n",
      "('grounded_emotions', 'isear', 'tweets', 'descriptions')\n",
      "('grounded_emotions', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "('grounded_emotions', 'emoint', 'tweets', 'tweets')\n",
      "('grounded_emotions', 'emobank', 'tweets', 'headlines')\n",
      "('grounded_emotions', 'dailydialog', 'tweets', 'conversations')\n",
      "('grounded_emotions', 'crowdflower', 'tweets', 'tweets')\n",
      "('grounded_emotions', 'affectivetext', 'tweets', 'headlines')\n",
      "('emoint', 'tec', 'tweets', 'tweets')\n",
      "('emoint', 'tales-emotion', 'tweets', 'tales')\n",
      "('emoint', 'ssec', 'tweets', 'tweets')\n",
      "('emoint', 'isear', 'tweets', 'descriptions')\n",
      "('emoint', 'grounded_emotions', 'tweets', 'tweets')\n",
      "('emoint', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "('emoint', 'emobank', 'tweets', 'headlines')\n",
      "('emoint', 'dailydialog', 'tweets', 'conversations')\n",
      "('emoint', 'crowdflower', 'tweets', 'tweets')\n",
      "('emoint', 'affectivetext', 'tweets', 'headlines')\n",
      "('crowdflower', 'tec', 'tweets', 'tweets')\n",
      "('crowdflower', 'tales-emotion', 'tweets', 'tales')\n",
      "('crowdflower', 'ssec', 'tweets', 'tweets')\n",
      "('crowdflower', 'isear', 'tweets', 'descriptions')\n",
      "('crowdflower', 'grounded_emotions', 'tweets', 'tweets')\n",
      "('crowdflower', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "('crowdflower', 'emoint', 'tweets', 'tweets')\n",
      "('crowdflower', 'emobank', 'tweets', 'headlines')\n",
      "('crowdflower', 'dailydialog', 'tweets', 'conversations')\n",
      "('crowdflower', 'affectivetext', 'tweets', 'headlines')\n",
      "('tales-emotion', 'tec', 'tales', 'tweets')\n",
      "('tales-emotion', 'ssec', 'tales', 'tweets')\n",
      "('tales-emotion', 'isear', 'tales', 'descriptions')\n",
      "('tales-emotion', 'grounded_emotions', 'tales', 'tweets')\n",
      "('tales-emotion', 'emotion-cause', 'tales', 'paragraphs')\n",
      "('tales-emotion', 'emoint', 'tales', 'tweets')\n",
      "('tales-emotion', 'emobank', 'tales', 'headlines')\n",
      "('tales-emotion', 'dailydialog', 'tales', 'conversations')\n",
      "('tales-emotion', 'crowdflower', 'tales', 'tweets')\n",
      "('tales-emotion', 'affectivetext', 'tales', 'headlines')\n",
      "('emotion-cause', 'tec', 'paragraphs', 'tweets')\n",
      "('emotion-cause', 'tales-emotion', 'paragraphs', 'tales')\n",
      "('emotion-cause', 'ssec', 'paragraphs', 'tweets')\n",
      "('emotion-cause', 'isear', 'paragraphs', 'descriptions')\n",
      "('emotion-cause', 'grounded_emotions', 'paragraphs', 'tweets')\n",
      "('emotion-cause', 'emoint', 'paragraphs', 'tweets')\n",
      "('emotion-cause', 'emobank', 'paragraphs', 'headlines')\n",
      "('emotion-cause', 'dailydialog', 'paragraphs', 'conversations')\n",
      "('emotion-cause', 'crowdflower', 'paragraphs', 'tweets')\n",
      "('emotion-cause', 'affectivetext', 'paragraphs', 'headlines')\n",
      "('emobank', 'tec', 'headlines', 'tweets')\n",
      "('emobank', 'tales-emotion', 'headlines', 'tales')\n",
      "('emobank', 'ssec', 'headlines', 'tweets')\n",
      "('emobank', 'isear', 'headlines', 'descriptions')\n",
      "('emobank', 'grounded_emotions', 'headlines', 'tweets')\n",
      "('emobank', 'emotion-cause', 'headlines', 'paragraphs')\n",
      "('emobank', 'emoint', 'headlines', 'tweets')\n",
      "('emobank', 'dailydialog', 'headlines', 'conversations')\n",
      "('emobank', 'crowdflower', 'headlines', 'tweets')\n",
      "('emobank', 'affectivetext', 'headlines', 'headlines')\n",
      "('affectivetext', 'tec', 'headlines', 'tweets')\n",
      "('affectivetext', 'tales-emotion', 'headlines', 'tales')\n",
      "('affectivetext', 'ssec', 'headlines', 'tweets')\n",
      "('affectivetext', 'isear', 'headlines', 'descriptions')\n",
      "('affectivetext', 'grounded_emotions', 'headlines', 'tweets')\n",
      "('affectivetext', 'emotion-cause', 'headlines', 'paragraphs')\n",
      "('affectivetext', 'emoint', 'headlines', 'tweets')\n",
      "('affectivetext', 'emobank', 'headlines', 'headlines')\n",
      "('affectivetext', 'dailydialog', 'headlines', 'conversations')\n",
      "('affectivetext', 'crowdflower', 'headlines', 'tweets')\n",
      "('isear', 'tec', 'descriptions', 'tweets')\n",
      "('isear', 'tales-emotion', 'descriptions', 'tales')\n",
      "('isear', 'ssec', 'descriptions', 'tweets')\n",
      "('isear', 'grounded_emotions', 'descriptions', 'tweets')\n",
      "('isear', 'emotion-cause', 'descriptions', 'paragraphs')\n",
      "('isear', 'emoint', 'descriptions', 'tweets')\n",
      "('isear', 'emobank', 'descriptions', 'headlines')\n",
      "('isear', 'dailydialog', 'descriptions', 'conversations')\n",
      "('isear', 'crowdflower', 'descriptions', 'tweets')\n",
      "('isear', 'affectivetext', 'descriptions', 'headlines')\n",
      "('dailydialog', 'tec', 'conversations', 'tweets')\n",
      "('dailydialog', 'tales-emotion', 'conversations', 'tales')\n",
      "('dailydialog', 'ssec', 'conversations', 'tweets')\n",
      "('dailydialog', 'isear', 'conversations', 'descriptions')\n",
      "('dailydialog', 'grounded_emotions', 'conversations', 'tweets')\n",
      "('dailydialog', 'emotion-cause', 'conversations', 'paragraphs')\n",
      "('dailydialog', 'emoint', 'conversations', 'tweets')\n",
      "('dailydialog', 'emobank', 'conversations', 'headlines')\n",
      "('dailydialog', 'crowdflower', 'conversations', 'tweets')\n",
      "('dailydialog', 'affectivetext', 'conversations', 'headlines')\n",
      "('affectivetext', 'affectivetext', 'headlines', 'headlines')\n",
      "('crowdflower', 'crowdflower', 'tweets', 'tweets')\n",
      "('dailydialog', 'dailydialog', 'conversations', 'conversations')\n",
      "('emobank', 'emobank', 'headlines', 'headlines')\n",
      "('emoint', 'emoint', 'tweets', 'tweets')\n",
      "('emotion-cause', 'emotion-cause', 'paragraphs', 'paragraphs')\n",
      "('grounded_emotions', 'grounded_emotions', 'tweets', 'tweets')\n",
      "('isear', 'isear', 'descriptions', 'descriptions')\n",
      "('ssec', 'ssec', 'tweets', 'tweets')\n",
      "('tales-emotion', 'tales-emotion', 'tales', 'tales')\n",
      "('tec', 'tec', 'tweets', 'tweets')\n",
      "(None, 'affectivetext', None, 'headlines')\n",
      "(None, 'crowdflower', None, 'tweets')\n",
      "(None, 'dailydialog', None, 'conversations')\n",
      "(None, 'emobank', None, 'headlines')\n",
      "(None, 'emoint', None, 'tweets')\n",
      "(None, 'emotion-cause', None, 'paragraphs')\n",
      "(None, 'grounded_emotions', None, 'tweets')\n",
      "(None, 'isear', None, 'descriptions')\n",
      "(None, 'ssec', None, 'tweets')\n",
      "(None, 'tales-emotion', None, 'tales')\n",
      "(None, 'tec', None, 'tweets')\n",
      "('tec', 'tales-emotion', 'tweets', 'tales')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  tales-emotion\n",
      "there were  14771  entries that were in test and  206668 that were not in test and  21051  that were in train\n",
      "test was appended  14771  times\n",
      "single\n",
      "oof\n",
      "Detected mode: single...\n",
      "21051 14771\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['anger', 'joy', 'noemo', 'surprise', 'disgust', 'sadness', 'fear']\n",
      "Making arrays\n",
      "checking for save files\n",
      "saved train_xNP as tec_tales-emotiontrain_xNP.npy\n",
      "saved train_yNP as tec_tales-emotiontrain_yNP.npy\n",
      "saved test_xNP as tec_tales-emotiontest_xNP.npy\n",
      "saved test_yNP as tec_tales-emotiontest_yNP.npy\n",
      "loading from np\n",
      "loaded directly from NP.load\n",
      "train_xNPSize (text) size loaded: 421.02 megabytes\n",
      "train_yNPSize (labels) size loaded: 0.084204 megabytes\n",
      "test_xNPSize (text) size loaded: 295.42 megabytes\n",
      "test_yNPSize (labels) size loaded: 0.059084 megabytes\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "True\n",
      "Loading classifier from file\n",
      "classifier loaded successfully\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.12659941777807868\n",
      "Recall\t0.12659941777807868\n",
      "F1-score\t0.12659941777807868\n",
      "Accuracy\t0.12659941777807868\n",
      "classifier already saved\n",
      "Total: 475 GiB\n",
      "Used: 292 GiB\n",
      "Free: 182 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'ssec', 'tweets', 'tweets')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test  ssec\n",
      "there were  4868  entries that were in test and  216571 that were not in test and  21051  that were in train\n",
      "test was appended  4868  times\n",
      "single\n",
      "Detected mode: multi...\n",
      "21051 4868\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['anger', 'joy', 'surprise', 'disgust', 'sadness', 'fear']\n",
      "Making arrays\n",
      "checking for save files\n",
      "saved train_xNP as tec_ssectrain_xNP.npy\n",
      "saved train_yNP as tec_ssectrain_yNP.npy\n",
      "saved test_xNP as tec_ssectest_xNP.npy\n",
      "saved test_yNP as tec_ssectest_yNP.npy\n",
      "loading from np\n",
      "loaded directly from NP.load\n",
      "train_xNPSize (text) size loaded: 421.02 megabytes\n",
      "train_yNPSize (labels) size loaded: 0.505224 megabytes\n",
      "test_xNPSize (text) size loaded: 97.36 megabytes\n",
      "test_yNPSize (labels) size loaded: 0.116832 megabytes\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "False\n",
      "file not found, creating new classifier\n",
      "this is the classifierName:  RandomForestClassifier\n",
      "Training...\n",
      "train_x (text) size: 421.02 megabytes\n",
      "train_y (labels) size: 0.505224 megabytes\n",
      "train_x (text) length: 21051\n",
      "train_y (labels) length: 21051\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 1 ... 0 0 0]]\n",
      "[[0 1 0 0 0 0]\n",
      " [0 0 0 0 1 0]\n",
      " [0 0 0 0 0 1]\n",
      " [0 0 0 0 0 1]\n",
      " [0 0 1 0 0 0]]\n",
      "finished training, classifier size: 4.8e-05 megabytes\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.5476020042949177\n",
      "Recall\t0.06002824858757062\n",
      "F1-score\t0.10819602574075385\n",
      "Accuracy\t0.09654889071487263\n",
      "classiferSaveFile:  tec_ssecRandomForestClassifier.pkl\n",
      "Saved Successfully\n",
      "Total: 475 GiB\n",
      "Used: 293 GiB\n",
      "Free: 181 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'isear', 'tweets', 'descriptions')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  isear\n",
      "there were  7666  entries that were in test and  213773 that were not in test and  21051  that were in train\n",
      "test was appended  7666  times\n",
      "single\n",
      "oof\n",
      "Detected mode: single...\n",
      "21051 7666\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['anger', 'joy', 'noemo', 'disgust', 'sadness', 'fear']\n",
      "Making arrays\n",
      "checking for save files\n",
      "emotions in make_arrays:  {'anger': 0, 'joy': 1, 'noemo': 2, 'disgust': 3, 'sadness': 4, 'fear': 5}\n",
      "train super raw:  0.178016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 21051/21051 [00:09<00:00, 2291.43it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 7666/7666 [00:03<00:00, 2128.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x length  21051\n",
      "train_x dimension of element  5000\n",
      "train_x (text) size RAW: 0.178016 megabytes\n",
      "train_y (labels) size RAW: 0.178016 megabytes\n",
      "test_x (text) size RAW: 0.061424 megabytes\n",
      "test_y (labels) size RAW: 0.061424 megabytes\n",
      "saved test_y\n",
      "train_x Size stays the same False\n",
      "train_y Size stays the same False\n",
      "test_x Size stays the same False\n",
      "test_y Size stays the same False\n",
      "train_xNPSize (text) size: 421.02 megabytes\n",
      "train_yNPSize (labels) size: 0.084204 megabytes\n",
      "test_xNPSize (text) size: 153.32 megabytes\n",
      "test_yNPSize (labels) size: 0.030664 megabytes\n",
      "train_xNP length  21051\n",
      "train_xNP dimension of element  2\n",
      "train_xNP size  105255000\n",
      "saving NP arrays\n",
      "NP arrays saved\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "False\n",
      "file not found, creating new classifier\n",
      "this is the classifierName:  RandomForestClassifier\n",
      "Training...\n",
      "train_x (text) size: 421.02 megabytes\n",
      "train_y (labels) size: 0.084204 megabytes\n",
      "train_x (text) length: 21051\n",
      "train_y (labels) length: 21051\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 1 ... 0 0 0]]\n",
      "[2 4 1 1 3]\n",
      "finished training, classifier size: 4.8e-05 megabytes\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.24002087138012002\n",
      "Recall\t0.24002087138012002\n",
      "F1-score\t0.24002087138012002\n",
      "Accuracy\t0.24002087138012002\n",
      "classiferSaveFile:  tec_isearRandomForestClassifier.pkl\n",
      "Saved Successfully\n",
      "Total: 475 GiB\n",
      "Used: 293 GiB\n",
      "Free: 181 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'grounded_emotions', 'tweets', 'tweets')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  grounded_emotions\n",
      "there were  2585  entries that were in test and  218854 that were not in test and  21051  that were in train\n",
      "test was appended  2585  times\n",
      "single\n",
      "oof\n",
      "Detected mode: single...\n",
      "21051 2585\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['joy', 'noemo', 'sadness']\n",
      "Making arrays\n",
      "checking for save files\n",
      "emotions in make_arrays:  {'joy': 0, 'noemo': 1, 'sadness': 2}\n",
      "train super raw:  0.178016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 21051/21051 [00:07<00:00, 2784.57it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 2585/2585 [00:00<00:00, 2920.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x length  21051\n",
      "train_x dimension of element  5000\n",
      "train_x (text) size RAW: 0.178016 megabytes\n",
      "train_y (labels) size RAW: 0.178016 megabytes\n",
      "test_x (text) size RAW: 0.02104 megabytes\n",
      "test_y (labels) size RAW: 0.02104 megabytes\n",
      "saved test_y\n",
      "train_x Size stays the same False\n",
      "train_y Size stays the same False\n",
      "test_x Size stays the same False\n",
      "test_y Size stays the same False\n",
      "train_xNPSize (text) size: 421.02 megabytes\n",
      "train_yNPSize (labels) size: 0.084204 megabytes\n",
      "test_xNPSize (text) size: 51.7 megabytes\n",
      "test_yNPSize (labels) size: 0.01034 megabytes\n",
      "train_xNP length  21051\n",
      "train_xNP dimension of element  2\n",
      "train_xNP size  105255000\n",
      "saving NP arrays\n",
      "NP arrays saved\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "False\n",
      "file not found, creating new classifier\n",
      "this is the classifierName:  RandomForestClassifier\n",
      "Training...\n",
      "train_x (text) size: 421.02 megabytes\n",
      "train_y (labels) size: 0.084204 megabytes\n",
      "train_x (text) length: 21051\n",
      "train_y (labels) length: 21051\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 1 ... 0 0 0]]\n",
      "[1 2 0 0 1]\n",
      "finished training, classifier size: 4.8e-05 megabytes\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.13152804642166344\n",
      "Recall\t0.13152804642166344\n",
      "F1-score\t0.13152804642166344\n",
      "Accuracy\t0.13152804642166344\n",
      "classiferSaveFile:  tec_grounded_emotionsRandomForestClassifier.pkl\n",
      "Saved Successfully\n",
      "Total: 475 GiB\n",
      "Used: 294 GiB\n",
      "Free: 180 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'emotion-cause', 'tweets', 'paragraphs')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  emotion-cause\n",
      "there were  2414  entries that were in test and  219025 that were not in test and  21051  that were in train\n",
      "test was appended  2414  times\n",
      "single\n",
      "oof\n",
      "Detected mode: single...\n",
      "21051 2414\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['anger', 'joy', 'noemo', 'surprise', 'disgust', 'sadness', 'fear']\n",
      "Making arrays\n",
      "checking for save files\n",
      "emotions in make_arrays:  {'anger': 0, 'joy': 1, 'noemo': 2, 'surprise': 3, 'disgust': 4, 'sadness': 5, 'fear': 6}\n",
      "train super raw:  0.178016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 21051/21051 [00:07<00:00, 2680.67it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 2414/2414 [00:00<00:00, 2828.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x length  21051\n",
      "train_x dimension of element  5000\n",
      "train_x (text) size RAW: 0.178016 megabytes\n",
      "train_y (labels) size RAW: 0.178016 megabytes\n",
      "test_x (text) size RAW: 0.02104 megabytes\n",
      "test_y (labels) size RAW: 0.02104 megabytes\n",
      "saved test_y\n",
      "train_x Size stays the same False\n",
      "train_y Size stays the same False\n",
      "test_x Size stays the same False\n",
      "test_y Size stays the same False\n",
      "train_xNPSize (text) size: 421.02 megabytes\n",
      "train_yNPSize (labels) size: 0.084204 megabytes\n",
      "test_xNPSize (text) size: 48.28 megabytes\n",
      "test_yNPSize (labels) size: 0.009656 megabytes\n",
      "train_xNP length  21051\n",
      "train_xNP dimension of element  2\n",
      "train_xNP size  105255000\n",
      "saving NP arrays\n",
      "NP arrays saved\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "False\n",
      "file not found, creating new classifier\n",
      "this is the classifierName:  RandomForestClassifier\n",
      "Training...\n",
      "train_x (text) size: 421.02 megabytes\n",
      "train_y (labels) size: 0.084204 megabytes\n",
      "train_x (text) length: 21051\n",
      "train_y (labels) length: 21051\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 1 ... 0 0 0]]\n",
      "[3 5 1 1 4]\n",
      "finished training, classifier size: 4.8e-05 megabytes\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.24026512013256007\n",
      "Recall\t0.24026512013256007\n",
      "F1-score\t0.24026512013256007\n",
      "Accuracy\t0.24026512013256007\n",
      "classiferSaveFile:  tec_emotion-causeRandomForestClassifier.pkl\n",
      "Saved Successfully\n",
      "Total: 475 GiB\n",
      "Used: 295 GiB\n",
      "Free: 179 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'emoint', 'tweets', 'tweets')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  emoint\n",
      "there were  7102  entries that were in test and  214337 that were not in test and  21051  that were in train\n",
      "test was appended  7102  times\n",
      "single\n",
      "oof\n",
      "Detected mode: single...\n",
      "21051 7102\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "['anger', 'joy', 'noemo', 'sadness', 'fear']\n",
      "Making arrays\n",
      "checking for save files\n",
      "emotions in make_arrays:  {'anger': 0, 'joy': 1, 'noemo': 2, 'sadness': 3, 'fear': 4}\n",
      "train super raw:  0.178016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 21051/21051 [00:18<00:00, 1157.40it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 7102/7102 [00:06<00:00, 1022.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x length  21051\n",
      "train_x dimension of element  5000\n",
      "train_x (text) size RAW: 0.178016 megabytes\n",
      "train_y (labels) size RAW: 0.178016 megabytes\n",
      "test_x (text) size RAW: 0.061424 megabytes\n",
      "test_y (labels) size RAW: 0.061424 megabytes\n",
      "saved test_y\n",
      "train_x Size stays the same False\n",
      "train_y Size stays the same False\n",
      "test_x Size stays the same False\n",
      "test_y Size stays the same False\n",
      "train_xNPSize (text) size: 421.02 megabytes\n",
      "train_yNPSize (labels) size: 0.084204 megabytes\n",
      "test_xNPSize (text) size: 142.04 megabytes\n",
      "test_yNPSize (labels) size: 0.028408 megabytes\n",
      "train_xNP length  21051\n",
      "train_xNP dimension of element  2\n",
      "train_xNP size  105255000\n",
      "saving NP arrays\n",
      "NP arrays saved\n",
      "Initializing classifier\n",
      "Searching for a  RandomForestClassifier\n",
      "False\n",
      "file not found, creating new classifier\n",
      "this is the classifierName:  RandomForestClassifier\n",
      "Training...\n",
      "train_x (text) size: 421.02 megabytes\n",
      "train_y (labels) size: 0.084204 megabytes\n",
      "train_x (text) length: 21051\n",
      "train_y (labels) length: 21051\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 1 1 ... 0 0 0]]\n",
      "[2 3 1 1 2]\n",
      "finished training, classifier size: 4.8e-05 megabytes\n",
      "Predicting...\n",
      "Analysing...\n",
      "analyse_results\n",
      "hello\n",
      "Precision\t0.2682343001971276\n",
      "Recall\t0.2682343001971276\n",
      "F1-score\t0.2682343001971276\n",
      "Accuracy\t0.2682343001971276\n",
      "classiferSaveFile:  tec_emointRandomForestClassifier.pkl\n",
      "Saved Successfully\n",
      "Total: 475 GiB\n",
      "Used: 295 GiB\n",
      "Free: 179 GiB\n",
      "-----------------------------------------------------------------------------------------\n",
      "('tec', 'emobank', 'tweets', 'headlines')\n",
      "Getting data\n",
      "get_train_test param:\n",
      "json  unified-dataset.jsonl\n",
      "train  tec\n",
      "test  emobank\n",
      "there were  10062  entries that were in test and  211377 that were not in test and  21051  that were in train\n",
      "test was appended  10062  times\n",
      "single\n",
      "Detected mode: multi...\n",
      "21051 10062\n",
      "Getting wordlist...\n",
      "Getting emotions\n",
      "[]\n",
      "Making arrays\n",
      "checking for save files\n",
      "emotions in make_arrays:  {}\n",
      "train super raw:  0.178016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 21051/21051 [00:11<00:00, 1864.99it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 10062/10062 [00:05<00:00, 1924.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x length  21051\n",
      "train_x dimension of element  5000\n",
      "train_x (text) size RAW: 0.178016 megabytes\n",
      "train_y (labels) size RAW: 0.178016 megabytes\n",
      "test_x (text) size RAW: 0.087616 megabytes\n",
      "test_y (labels) size RAW: 0.087616 megabytes\n",
      "saved test_y\n",
      "train_x Size stays the same False\n",
      "train_y Size stays the same False\n",
      "test_x Size stays the same False\n",
      "test_y Size stays the same False\n",
      "train_xNPSize (text) size: 421.02 megabytes\n",
      "train_yNPSize (labels) size: 0.0 megabytes\n",
      "test_xNPSize (text) size: 201.24 megabytes\n",
      "test_yNPSize (labels) size: 0.0 megabytes\n",
      "train_xNP length  21051\n",
      "train_xNP dimension of element  2\n",
      "train_xNP size  105255000\n",
      "Train or test empty. Did you misspell the dataset name?\n",
      "End of program!\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "#     args = docopt.docopt(__doc__, version=\"0.0.1\")\n",
    "#     args = {'--all-vs': True,\n",
    "#      '--debug': True,\n",
    "#      '--force-multi': False,\n",
    "#      '--json': 'unified-dataset.jsonl',\n",
    "#      '--output': '.',\n",
    "#      '<first>': 'tec',\n",
    "#      '<second>': 'affectivetext'}\n",
    "#     print(args)\n",
    "    \n",
    "    possibleChoices = [('affectivetext','headlines'), ('crowdflower','tweets'), ('dailydialog','conversations'), \n",
    "                       ('emobank','headlines'), ('emoint','tweets'), \n",
    "                       ('emotion-cause','paragraphs'), ('grounded_emotions','tweets'), ('isear','descriptions'),\n",
    "                       ('ssec','tweets'),('tales-emotion','tales'), ('tec','tweets')] #('electoraltweets','tweets') <-run trials with this later\n",
    "    \n",
    "    #     print(possibleChoices)\n",
    "    permutations = list(getPermutations(possibleChoices))\n",
    "    powerSet = list(getPowerset(possibleChoices))\n",
    "#     print(\"powerset: \", powerSet)\n",
    "    print(\"permutations length: \",len(permutations))\n",
    "#     print(permutations)\n",
    "    corporaSets = []\n",
    "    for choice in permutations:\n",
    "#         print(\"choice \", choice)\n",
    "        if(len(choice) == 2):\n",
    "#             print(\"pair\")\n",
    "            first, second = choice\n",
    "            firstCorpus, domain1 = first\n",
    "            secondCorpus, domain2 = second\n",
    "#         else:\n",
    "# #             print(\"less than 2\")\n",
    "#             firstCorpus, domain1 = choice[0]\n",
    "#             secondCorpus, domain2 = choice[0] #repeat\n",
    "        corpusPairData = (firstCorpus, secondCorpus, domain1, domain2)\n",
    "        corporaSets.append(corpusPairData)\n",
    "#     print(corporaSets)\n",
    "    sortedPermutations = sorted(corporaSets, key = lambda x: (x[2], x[0], x[1]), reverse = True)\n",
    "    for entry in possibleChoices:\n",
    "        firstCorpus, domain1 = entry\n",
    "        secondCorpus, domain2 = entry\n",
    "        corpusPairData = (firstCorpus, secondCorpus, domain1, domain2)\n",
    "        sortedPermutations.append(corpusPairData)\n",
    "    for entry in possibleChoices:\n",
    "        firstCorpus, domain1 = (None, None)\n",
    "        secondCorpus, domain2 = entry\n",
    "        corpusPairData = (firstCorpus, secondCorpus, domain1, domain2)\n",
    "        sortedPermutations.append(corpusPairData)\n",
    "    for entry in sortedPermutations:\n",
    "        print(entry)\n",
    "#     powerSetCondensedGood = []\n",
    "#     powerSetCondensedBad = []\n",
    "#     powerSetCondensed = []\n",
    "#     for entry in powerSet:\n",
    "# #         if len(entry) < 3 and len(entry) > 0:\n",
    "#             if len(entry) == 2:\n",
    "#                 domainMatch = entry[0][1]\n",
    "#                 appendGood = True\n",
    "#                 for corpus, domain in entry:\n",
    "#                     if domain != domainMatch:\n",
    "#                         appendGood = False\n",
    "#                 if(appendGood):\n",
    "#                     print(\"good entry\",entry)\n",
    "#                     powerSetCondensedGood.append(entry)\n",
    "#                 else:\n",
    "#                     print(\"bad entry\",entry)\n",
    "#                     powerSetCondensedBad.append(entry)\n",
    "#             if len(entry) < 3 and len(entry) > 0:\n",
    "#                 powerSetCondensed.append(entry)\n",
    "#     print(\"powerSetCondensed length: \",len(powerSetCondensed))\n",
    "#     print(\"powerSetCondensedGood length: \",len(powerSetCondensedGood))\n",
    "#     print(\"powerSetCondensedGood: \", powerSetCondensedGood)\n",
    "#     print(\"powerSetCondensedBad length: \",len(powerSetCondensedBad))\n",
    "#     print(\"powerSetCondensedBad: \", powerSetCondensedBad)\n",
    "#     print(powerSetCondensedGood[0])\n",
    "    \n",
    "#     example1 = ('ssec', 'tec', 'tweets', 'tweets')\n",
    "#     example2 = (None, 'affectivetext', None, 'headlines')\n",
    "    for entry in sortedPermutations:\n",
    "        print(entry)\n",
    "        (first, second, domain1, domain2) = entry\n",
    "        print(\"Getting data\")\n",
    "        jsonfile = \"unified-dataset.jsonl\"\n",
    "#         first = example2[0] #use first = None if you want to do ALl vs\n",
    "#         second = example2[1]\n",
    "    #     first = \"isear\" #use first = None if you want to do ALl vs\n",
    "    #     second = \"crowdflower\"\n",
    "\n",
    "        output = \".\"\n",
    "        debug = True\n",
    "        forceMulti = False\n",
    "        isAllVS = False\n",
    "        if first == None:\n",
    "            isAllVS = True\n",
    "\n",
    "        training_data, testing_data = get_train_test(jsonfile, first,second)\n",
    "        firstCLF, secondCLF = ([\"multi\", \"multi\"] if forceMulti else get_clf_mode(training_data, testing_data))\n",
    "        mode = \"multi\" if \"multi\" in [firstCLF, secondCLF] else \"single\"\n",
    "\n",
    "        print(\"Detected mode: {}...\".format(mode))\n",
    "        print(len(training_data), len(testing_data))\n",
    "        print(\"Getting wordlist...\")\n",
    "        if debug:\n",
    "            wordlist = get_wordlist_debug(training_data)\n",
    "        else:\n",
    "            wordlist = get_wordlist_debug(training_data)\n",
    "            # wordlist = get_wordlist(training_data)\n",
    "        print(\"Getting emotions\")\n",
    "        labels = get_labels(training_data, testing_data, mode=mode)\n",
    "        print(labels)\n",
    "        print(\"Making arrays\")\n",
    "        print(\"checking for save files\")\n",
    "        if(first == None):\n",
    "            first = \"all-vs\"\n",
    "        train_xNPFileName = first + \"_\" + second + \"train_xNP\" +\".npy\"\n",
    "        train_yNPFileName = first + \"_\" + second + \"train_yNP\" +\".npy\"\n",
    "        test_xNPFileName = first + \"_\" + second + \"test_xNP\" +\".npy\"\n",
    "        test_yNPFileName = first + \"_\" + second + \"test_yNP\" +\".npy\"\n",
    "\n",
    "        if(path.exists(train_xNPFileName) \n",
    "           and path.exists(train_yNPFileName)\n",
    "           and path.exists(test_xNPFileName)\n",
    "           and path.exists(test_yNPFileName)):\n",
    "            print('saved train_xNP as', train_xNPFileName)\n",
    "            print('saved train_yNP as', train_yNPFileName)\n",
    "            print('saved test_xNP as', test_xNPFileName)\n",
    "            print('saved test_yNP as', test_yNPFileName)\n",
    "            print(\"loading from np\")\n",
    "            train_x = np.load(train_xNPFileName)\n",
    "            train_y = np.load(train_yNPFileName)\n",
    "            test_x = np.load(test_xNPFileName)\n",
    "            test_y = np.load(test_yNPFileName)\n",
    "            train_xNPSize = (train_x.nbytes)/1000000\n",
    "            train_yNPSize = (train_y.nbytes)/1000000\n",
    "            test_xNPSize = (test_x.nbytes)/1000000\n",
    "            test_yNPSize = (test_y.nbytes)/1000000\n",
    "            print(\"loaded directly from NP.load\")\n",
    "            print(\"train_xNPSize (text) size loaded:\", train_xNPSize,\"megabytes\")\n",
    "            print(\"train_yNPSize (labels) size loaded:\", train_yNPSize,\"megabytes\")\n",
    "            print(\"test_xNPSize (text) size loaded:\", test_xNPSize,\"megabytes\")\n",
    "            print(\"test_yNPSize (labels) size loaded:\", test_yNPSize,\"megabytes\")\n",
    "        else:\n",
    "            train_x, train_y, test_x, test_y, sizes = make_arrays(training_data, testing_data, wordlist, labels, mode, isAllVS)\n",
    "            train_xSize, train_ySize, test_xSize, test_ySize = sizes\n",
    "            if any(not part.size for part in [train_x, train_y, test_x, test_y]):\n",
    "                print(\"Train or test empty. Did you misspell the dataset name?\")\n",
    "                continue\n",
    "            #             sys.exit(1)\n",
    "            print(\"saving NP arrays\")\n",
    "            np.save(train_xNPFileName, train_x)\n",
    "            np.save(train_yNPFileName, train_y)\n",
    "            np.save(test_xNPFileName, test_x)\n",
    "            np.save(test_yNPFileName, test_y)\n",
    "    #         joblib.dump(train_x, train_xNPFileName)\n",
    "    #         joblib.dump(train_y, train_yNPFileName)\n",
    "    #         joblib.dump(test_x, test_xNPFileName)\n",
    "    #         joblib.dump(test_y, test_yNPFileName)\n",
    "            print(\"NP arrays saved\")\n",
    "        \n",
    "        print(\"Initializing classifier\")\n",
    "        trainClassifier = True\n",
    "        if debug:\n",
    "            classifierName = \"RandomForestClassifier\"\n",
    "            print(\"Searching for a \", classifierName)\n",
    "            classiferSaveFile = first+\"_\"+second+classifierName+\".pkl\"\n",
    "            print(path.exists(classiferSaveFile))\n",
    "            if(path.exists(classiferSaveFile)):\n",
    "                trainClassifier = False\n",
    "                print(\"Loading classifier from file\")\n",
    "                classifier = joblib.load(classiferSaveFile)\n",
    "                print(\"classifier loaded successfully\")\n",
    "            else:\n",
    "                print(\"file not found, creating new classifier\")\n",
    "                classifier = RandomForestClassifier()\n",
    "        elif mode == \"single\":\n",
    "            classifierName = \"LogisticRegressionCV\"\n",
    "            print(\"Searching for a \", classifierName)\n",
    "            classiferSaveFile = first+\"_\"+second+classifierName+\".pkl\"\n",
    "            print(path.exists(classiferSaveFile))\n",
    "            if(path.exists(classiferSaveFile)):\n",
    "                trainClassifier = False\n",
    "                print(\"Loading classifier from file\")\n",
    "                classifier = joblib.load(classiferSaveFile)\n",
    "                print(\"classifier loaded successfully\")\n",
    "            else:\n",
    "                print(\"file not found, creating new classifier\")\n",
    "                classifier = LogisticRegressionCV(\n",
    "                    cv=10,\n",
    "                    penalty=\"l2\",\n",
    "                    fit_intercept=True,\n",
    "                    solver=\"sag\",\n",
    "                    scoring=\"f1\",\n",
    "                    refit=True,\n",
    "                    # n_jobs=-1,\n",
    "                    class_weight=\"balanced\",\n",
    "                )\n",
    "        else:\n",
    "            classifierName = \"OneVsRestClassifier\"\n",
    "            print(\"Searching for a \", classifierName)\n",
    "            classiferSaveFile = first+\"_\"+second+classifierName+\".pkl\"\n",
    "            print(path.exists(classiferSaveFile))\n",
    "            if(path.exists(classiferSaveFile)):\n",
    "                trainClassifier = False\n",
    "                print(\"Loading classifier from file\")\n",
    "                classifier = joblib.load(classiferSaveFile)\n",
    "                print(\"classifier loaded successfully\")\n",
    "            else:\n",
    "                print(\"file not found, creating new classifier\")\n",
    "                classifier = OneVsRestClassifier(\n",
    "                    LogisticRegressionCV(\n",
    "                        cv=10,\n",
    "                        penalty=\"l2\",\n",
    "                        fit_intercept=True,\n",
    "                        solver=\"sag\",\n",
    "                        scoring=\"f1\",\n",
    "                        refit=True,\n",
    "                        class_weight=\"balanced\",\n",
    "                        tol = 0.1,\n",
    "                    ),\n",
    "                    n_jobs=-1,\n",
    "                )\n",
    "        if(trainClassifier):\n",
    "            print(\"this is the classifierName: \", classifierName)\n",
    "            print(\"Training...\")\n",
    "            print(\"train_x (text) size:\", (train_x.nbytes)/1000000,\"megabytes\")\n",
    "            print(\"train_y (labels) size:\", (train_y.nbytes)/1000000,\"megabytes\")\n",
    "            print(\"train_x (text) length:\", len(train_x))\n",
    "            print(\"train_y (labels) length:\", len(train_y))\n",
    "            print(train_x[:5])\n",
    "            print(train_y[:5])\n",
    "\n",
    "            classifier.fit(train_x, train_y)\n",
    "            print(\"finished training, classifier size:\", sys.getsizeof(classifier)/1000000,\"megabytes\")\n",
    "        print(\"Predicting...\")\n",
    "        if first == \"multi\" and second == \"single\":\n",
    "            predict_y = classifier.predict_proba(test_x)\n",
    "            helper = np.zeros_like(predict_y)\n",
    "            helper[range(len(predict_y)), predict_y.argmax(1)] = 1\n",
    "            predict_y = helper\n",
    "        else:\n",
    "            predict_y = classifier.predict(test_x)\n",
    "\n",
    "        print(\"Analysing...\")\n",
    "\n",
    "        analyse_results(\n",
    "            test_y,\n",
    "            predict_y,\n",
    "            labels,\n",
    "            testing_data,\n",
    "            first,\n",
    "            second,\n",
    "            output,\n",
    "            mode,  # TODO\n",
    "        )\n",
    "        if(path.exists(classiferSaveFile)):\n",
    "            print(\"classifier already saved\")\n",
    "        else:\n",
    "    #         classiferSaveFile = first+\"_\"+second+classifierName+\".pkl\"\n",
    "            print(\"classiferSaveFile: \", classiferSaveFile)\n",
    "            joblib.dump(classifier, classiferSaveFile)\n",
    "            print(\"Saved Successfully\")\n",
    "        total, used, free = getHardDriveSpaceLeft()\n",
    "        if(free < 10):\n",
    "            sys.exit(\"Error: less than 10 gb remaining on disk\")\n",
    "        print(\"-----------------------------------------------------------------------------------------\")\n",
    "print(\"End of program!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = joblib.load(\"grounded_emotions_emointRandomForestClassifier.pkl\")\n",
    "stringEx = first+\"_\"+second+classifierName+\".pkl\"\n",
    "print(\"there is a classifier: \", classifier)\n",
    "analyse_results(\n",
    "        test_y,\n",
    "        predict_y,\n",
    "        labels,\n",
    "        testing_data,\n",
    "        first,\n",
    "        second,\n",
    "        output,\n",
    "        mode,  # TODO\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-94-c580fe045f86>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-94-c580fe045f86>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    get_train_test param:\u001b[0m\n\u001b[1;37m                   ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "get_train_test param:\n",
    "json  unified-dataset.jsonl\n",
    "train  grounded_emotions\n",
    "test  emoint\n",
    "there were  7102  entries that were in test and  214337 that were not in test and  2585  that were in train\n",
    "test was appended  7102  times\n",
    "single\n",
    "oof\n",
    "Detected mode: single...\n",
    "2585 7102\n",
    "Getting wordlist...\n",
    "Getting emotions\n",
    "['joy', 'noemo', 'sadness']\n",
    "Making arrays\n",
    "emotions in make_arrays:  {'joy': 0, 'noemo': 1, 'sadness': 2}\n",
    "train_x (text) size RAW: (0.02104, 'megabytes')\n",
    "train_y (labels) size RAW: (0.02104, 'megabytes')\n",
    "test_x (text) size RAW: (0.061424, 'megabytes')\n",
    "test_y (labels) size RAW: (0.061424, 'megabytes')\n",
    "Initializing classifier\n",
    "Searching for a  RandomForestClassifier\n",
    "Loading classifier from file\n",
    "classifier loaded successfully\n",
    "Predicting...\n",
    "Analysing...\n",
    "analyse_results\n",
    "hello\n",
    "Precision\t0.5167558434243875\n",
    "Recall\t0.5167558434243875\n",
    "F1-score\t0.5167558434243875\n",
    "Accuracy\t0.5167558434243875"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "there were  1250  entries that were in test and  220189 that were not in test and  2585  that were in train\n",
    "test was appended  1250  times\n",
    "single\n",
    "Detected mode: multi...\n",
    "2585 1250\n",
    "Getting wordlist...\n",
    "Getting emotions\n",
    "['sadness', 'joy']\n",
    "Making arrays\n",
    "emotions in make_arrays:  {'sadness': 0, 'joy': 1}\n",
    "train_x (text) size RAW: (0.02104, 'megabytes')\n",
    "train_y (labels) size RAW: (0.02104, 'megabytes')\n",
    "test_x (text) size RAW: (0.010192, 'megabytes')\n",
    "test_y (labels) size RAW: (0.010192, 'megabytes')\n",
    "Initializing classifier\n",
    "Searching for a  RandomForestClassifier\n",
    "False\n",
    "file not found, creating new classifier\n",
    "this is the classifierName:  RandomForestClassifier\n",
    "Training...\n",
    "train_x (text) size: 51.700112 megabytes\n",
    "train_y (labels) size: 0.020792 megabytes\n",
    "train_x (text) length: 2585\n",
    "train_y (labels) length: 2585"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x length 2585\n",
    "train_x dimension of element 5000\n",
    "train_x (text) size RAW: 0.02104 megabytes\n",
    "train_y (labels) size RAW: 0.02104 megabytes\n",
    "test_x (text) size RAW: 0.010192 megabytes\n",
    "test_y (labels) size RAW: 0.010192 megabytes\n",
    "train_x Size stays the same False\n",
    "train_x Size stays the same False\n",
    "train_x Size stays the same False\n",
    "train_x Size stays the same False\n",
    "train_xNPSize (text) size: 51.700112 megabytes\n",
    "train_yNPSize (labels) size: 0.020792 megabytes\n",
    "test_xNPSize (text) size: 25.000112 megabytes\n",
    "test_yNPSize (labels) size: 0.010112 megabytes\n",
    "Initializing classifier\n",
    "Searching for a  RandomForestClassifier\n",
    "True\n",
    "Loading classifier from file\n",
    "classifier loaded successfully\n",
    "Predicting...\n",
    "Analysing...\n",
    "analyse_results\n",
    "hello\n",
    "Precision\t0.4912\n",
    "Recall\t0.4838455476753349\n",
    "F1-score\t0.4874950377133783\n",
    "Accuracy\t0.3568\n",
    "classifier already saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi\n"
     ]
    }
   ],
   "source": [
    "print(\"hi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}