ACL_18_result_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = r'result_log\\PCIE\\finetune\\2411hftm_test.txt'  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pct = 'ft'\n",
    "if pct == 'pct':\n",
    "    pl_i = 3\n",
    "elif pct == 'ft':\n",
    "    pl_i = 4\n",
    "else:\n",
    "    pl_i = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pct': [array([ 7.50332754, 51.62415241]), 1000]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = open(file_path, 'r')\n",
    "\n",
    "lines = f.readlines()\n",
    "\n",
    "result_dict = {}\n",
    "\n",
    "check_limit = 1000\n",
    "\n",
    "i = 0\n",
    "ct = 0\n",
    "\n",
    "\n",
    "while i < len(lines) and ct < check_limit:\n",
    "\n",
    "    if len(lines[i].split('_')) > 5 :\n",
    "        \n",
    "        line_1 = lines[i].split('_')\n",
    "        line_2 = lines[i+1].split(', ')\n",
    "\n",
    "        pred_len = line_1[pl_i]\n",
    "        mse = np.float64(line_2[0].split(':')[1])\n",
    "        mae = np.float64(line_2[1].split(':')[1])\n",
    "\n",
    "        if pred_len not in result_dict:\n",
    "            result_dict[pred_len] = [np.zeros(2, dtype=np.float64), 0]  # mse, mae, ct\n",
    "\n",
    "        \n",
    "        result_dict[pred_len][0][0] += mse\n",
    "        result_dict[pred_len][0][1] += mae\n",
    "        result_dict[pred_len][1] += 1\n",
    "\n",
    "        # print(mse/result_dict[pred_len][1], mae/result_dict[pred_len][1], type(mse), type(mae), type(result_dict[pred_len][1]))\n",
    "        # print(result_dict[pred_len])\n",
    "\n",
    "\n",
    "        ct += 1\n",
    "        i += 3\n",
    "\n",
    "    else:\n",
    "        i += 1\n",
    "    \n",
    "f.close()\n",
    "\n",
    "result_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pct': [0.007503327544080093, 0.05162415241077542]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_result = {}\n",
    "for key in result_dict:\n",
    "    final_result[key] = [result_dict[key][0][0]/result_dict[key][1] , result_dict[key][0][1]/result_dict[key][1] ]\n",
    "\n",
    "final_result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pretrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'10': [1.0199270854953308, 0.7193449714653929],\n",
       " '20': [1.0089587029856695, 0.7175241847273329],\n",
       " '40': [0.9979861243300034, 0.7139840310728046],\n",
       " '60': [0.9896148657295066, 0.7116840343240282]}"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#2005 raw\n",
    "{'10': [0.13410979545173737, 0.24645035083477312]}\n",
    "{'20': [0.2732677317357489, 0.3585611933044025]}\n",
    "{'40': [0.5132464894226619, 0.49892272906644003]}\n",
    "{'60': [0.8081669360399246, 0.6311179857168879]} \n",
    "\n",
    "#2005 pct\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#2016 pct\n",
    "{'10': [1.0199270854953308, 0.7193449714653929],\n",
    " '20': [1.0089587029856695, 0.7175241847273329],\n",
    " '40': [0.9979861243300034, 0.7139840310728046],\n",
    " '60': [0.9896148657295066, 0.7116840343240282]}\n",
    "\n",
    "\n",
    "#2016 raw\n",
    "{'10': [0.06862954018985144, 0.177440764408716],\n",
    " '20': [0.13044368256126124, 0.24929575134121196],\n",
    " '40': [0.2384338671617, 0.34340609017182405],\n",
    " '60': [0.32082978725879335, 0.4022367992241618]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ablation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_l = [\n",
    "\n",
    "{'10': [1.5957395434379578, 0.8827536063534873],\n",
    " '20': [1.5637823130403246, 0.8834761679172516],\n",
    " '40': [1.5536138245037623, 0.8802967497280666],\n",
    " '60': [1.5417637271540505, 0.8786286881991795]} #v2 prediction\n",
    " \n",
    ",\n",
    "\n",
    "{'10': [0.17117775244904415, 0.28375182513679775],\n",
    " '20': [0.31789635135126965, 0.38963014845337185],\n",
    " '40': [0.6483245891119752, 0.5538921004959515],\n",
    " '60': [0.9454534538090229, 0.6681726032069751]}  #v2 raw\n",
    "\n",
    ",       \n",
    "\n",
    "{'10': [1.1056005304128351, 0.7354617937349938],\n",
    " '20': [1.0793213122327563, 0.7309196234588892],\n",
    " '40': [1.0455695133603795, 0.7208275522144747],\n",
    " '60': [1.1613256421307443, 0.7695941530482869]}    #mix pct   #d\n",
    ",\n",
    "\n",
    "{'10': [0.08380883581473977, 0.20038651811404967],\n",
    " '20': [0.15562962041020026, 0.2765597148313069],\n",
    " '40': [0.2964489205779744, 0.385121331196016],\n",
    " '60': [0.40310251655083307, 0.44580053181295665]}     #raw     #d\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## V2 raw log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'10': [0.15559919417968818, 0.2682269146399839],\n",
       " '20': [0.2942389579755919, 0.3735662634883608],\n",
       " '40': [0.5705482954425471, 0.524192462010043],\n",
       " '60': [0.8188239910772869, 0.6346315060343061]}"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{'10': [0.14586360326835088, 0.25904200013194767],\n",
    " '20': [0.2794788314827851, 0.36254675100956646],\n",
    " '40': [0.5569841332201447, 0.5203362362725394],    #NON XL\n",
    " '60': [0.8251644644354071, 0.6355077411447253]}   #EcmP\n",
    "\n",
    "\n",
    "{'10': [0.3009432550253613, 0.38808583095669746],\n",
    " '20': [0.4543194925146444, 0.47889087562050137],\n",
    " '40': [0.7498087228408882, 0.6274954942720277],\n",
    " '60': [0.9885579975587981, 0.7247902410370963]}   #Autoformer\n",
    "\n",
    "\n",
    "{'10': [28.688600864660526, 3.4267060134027685],\n",
    " '20': [27.124226781140482, 3.405829274228641],\n",
    " '40': [27.65633640864066, 3.5118626611573354],\n",
    " '60': [28.25760881709201, 3.6177804342338016]}    #informer\n",
    "\n",
    "\n",
    "{'10': [0.15559919417968818, 0.2682269146399839],\n",
    " '20': [0.2942389579755919, 0.3735662634883608],\n",
    " '40': [0.5705482954425471, 0.524192462010043],\n",
    " '60': [0.8188239910772869, 0.6346315060343061]}   #patchTST\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## V2 Pct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'10': [1.6414782106876373, 0.8844754781041827],\n",
       " '20': [1.575849290405001, 0.8764377193791526],\n",
       " '40': [1.5337467576776231, 0.8681245190756661],\n",
       " '60': [1.5116292876856667, 0.8638575545379094]}"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{'10': [1.572515836783818, 0.8721245867865426],\n",
    " '20': [1.5181358371462141, 0.8601870281355721],   #PCIE   mlp\n",
    " '40': [1.4746983647346497, 0.8520571121147701],\n",
    " '60': [1.4611012935638428, 0.8502018536840167]}   \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "{'10': [1.5920423907893044, 0.8756769725254604],    \n",
    " '20': [1.5457126923969813, 0.8692560749394553],\n",
    " '40': [1.5078203976154327, 0.8627119788101741],\n",
    " '60': [1.490694556917463, 0.8590175722326551]} #autoformer\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "{'10': [2.4028169895921434, 1.0675077395779746],\n",
    " '20': [2.277534548725401, 1.0455524793692998],\n",
    " '40': [1.5192493157727378, 0.8692139600004468],\n",
    " '60': [1.4562638231686182, 0.8485054118292672]}  #informer\n",
    "\n",
    "\n",
    "\n",
    "{'10': [1.6414782106876373, 0.8844754781041827],\n",
    " '20': [1.575849290405001, 0.8764377193791526],\n",
    " '40': [1.5337467576776231, 0.8681245190756661],\n",
    " '60': [1.5116292876856667, 0.8638575545379094]}  #patchtst"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mix raw log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "mix_raw_log_l = [\n",
    "{'10': [0.0690710063274412, 0.17839710500983286],   #64\n",
    "    '20': [0.13521061592873082, 0.25537825777182277],   #64\n",
    "    '40': [0.2613453817370177, 0.36178465182302705],  #xl p1 aba\n",
    "    '60': [0.333790597328785, 0.41562228314053845]} #xl  #ecmp v1\\\n",
    ",\n",
    "{'10': [0.07513540764388457, 0.18726823707176765],\n",
    "    '20': [0.1450061819985063, 0.2645496417256728],\n",
    "    '40': [0.2686248190576394, 0.3656771437487972],\n",
    "    '60': [0.35874927510887805, 0.42563069931847947]}   #PatchTST\\n\",\n",
    ",\n",
    "{'10': [0.1292227408914289, 0.25838516867706474],\n",
    "    '20': [0.2112721208430512, 0.3260804279379442],\n",
    "    '40': [0.3134443444973776, 0.4103433995599478],\n",
    "    '60': [0.38978398179399293, 0.4592954568879705]}   #Autoformer\n",
    ",\n",
    "{'10': [1.5269764208720185, 0.7560676116968544],\n",
    "    '20': [1.4828249227098176, 0.8014524073248178],\n",
    "    '40': [1.8022921134940755, 0.9289577861487026],\n",
    "    '60': [2.350615150124674, 1.0924766668131654]}    #Informer\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.30008516867706475\n",
      "0.3677804279379442\n",
      "0.4520433995599478\n",
      "0.5009954568879705\n"
     ]
    }
   ],
   "source": [
    "for k in mix_raw_log_l[2]:\n",
    "    print(mix_raw_log_l[2][k][1] + 0.0417)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mix pct log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "mix_pct = [\n",
    "{'10': [1.0026603622335784, 0.7108989446096017],\n",
    " '20': [0.9961253740418126, 0.7105563350126777],   \n",
    " '40': [0.9792992795856905, 0.7035276260174496], \n",
    " '60': [0.9983368763621424, 0.715730220079422]}  #revin on    #aba\n",
    ",\n",
    "\n",
    "{'10': [1.0660486924396435, 0.7405942384625824],\n",
    " '20': [1.0570164247297904, 0.7369471007669476],\n",
    " '40': [1.0272659248029683, 0.725262936571954],\n",
    " '60': [1.013876146623786, 0.720789264205476]}   #PatchTST\n",
    ",\n",
    "{'10': [1.1650539890561304, 0.7729455064719831],\n",
    " '20': [1.1484186477224592, 0.7730349801795583],\n",
    " '40': [1.0878558381342551, 0.751602764280749],\n",
    " '60': [1.0882896162884337, 0.7450747968445361]}   #Informer\n",
    ",\n",
    "{'10': [1.0144202757469365, 0.7163188276996075],\n",
    " '20': [1.0140405419007155, 0.7172929170266004],\n",
    " '40': [1.0039774057730821, 0.7142338584846174],\n",
    " '60': [0.9908792795219892, 0.7100594853851158]}  #Autoformer  #td\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RAW Log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'result_log\\EcmP_mk3\\ecmp_stock\\final_10_40_v1_dff_t2.txt'\n",
    "# 'result_log\\EcmP_mk3\\ecmp_stock\\final_seq_60_v1.txt'\n",
    "raw_log_l = [\n",
    "\n",
    "{'10': [0.06986033357515879, 0.1797839212165752],\n",
    " '20': [0.13419161343776528, 0.25539891547720195],\n",
    " '40': [0.2553188940608533, 0.35963834990078297],\n",
    " '60': [0.33873793937292107, 0.4185521456242447]}    #EcmP mk3 Linear None\n",
    "\n",
    ",\n",
    "\n",
    "# 'result_log\\PatchTST\\ecmp_stock\\test_1.txt'\n",
    "{'10': [0.0749773853860179, 0.18674399008528447],\n",
    " '20': [0.14426159017889853, 0.2641829145993565],\n",
    " '40': [0.26221268738842973, 0.3632862289291872],\n",
    " '60': [0.34698844791232597, 0.41897929538513573]}     #Patch TST v1\n",
    ",\n",
    "\n",
    "# 'result_log\\Autoformer\\ecmp_stock.txt'\n",
    "{'10': [0.13350610889520653, 0.25984287671220135],\n",
    " '20': [0.20200920743789053, 0.3253066968959822],\n",
    " '40': [0.32407858315855265, 0.41575249232036965],\n",
    " '60': [0.39311297390032823, 0.46052118632155403]}     #Autoformer \n",
    ",\n",
    "\n",
    "# 'result_log\\Informer\\ecmp_stock.txt'\n",
    "{'10': [1.224894627640155, 0.6743516214716603],\n",
    " '20': [1.4129749486668848, 0.7523451308969041],\n",
    " '40': [1.6858338479110053, 0.8610608063533273],\n",
    " '60': [2.022184497847314, 0.9609464230671735]}        #Informer\n",
    "\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCT log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "pct_log = [\n",
    "\n",
    "{'10': [1.0125709467790496, 0.7109686984142787],\n",
    " '20': [1.0449350791078218, 0.7153248992604269],\n",
    " '40': [1.068168761654639, 0.712965254212769],   #ECMP  revin off  result_log\\EcmP_mk3\\ecmp_stock\\final_10_40_pct_xl_s40_revin.txt\n",
    " '60': [1.0595059113603242, 0.7419622460721245]}  #ECMP  revin on  result_log\\EcmP_mk3\\ecmp_stock\\final_seq_60_pct_xl.txt\n",
    "\n",
    ",\n",
    "{'10': [1.080351742010721, 0.7463258525855104],\n",
    " '20': [1.0773388634265308, 0.7467892169952393],\n",
    " '40': [1.0518272506519102, 0.7369517474107339],\n",
    " '60': [1.0426115148084265, 0.7341517646547774]}  #PatchTST \n",
    ",\n",
    "{'10': [1.029358135352672, 0.7205219587809603],\n",
    " '20': [1.0243071256808831, 0.7196187041175197],\n",
    " '40': [1.0057641563700959, 0.7144720067440624],\n",
    " '60': [0.9948612420072018, 0.7109915824003623]}  #Autoformer\n",
    ",\n",
    "\n",
    "{'10': [0.983496680855751, 0.7043920767978883],\n",
    " '20': [0.981861832066321, 0.7055761104738209],\n",
    " '40': [0.9671040012593001, 0.6993108450526923],\n",
    " '60': [0.9620522795848443, 0.6982525561896848]}  #informer\n",
    "\n",
    "\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## improvement comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_result = []\n",
    "\n",
    "#a - b\n",
    "a = pct_log    \n",
    "\n",
    "b = mix_pct\n",
    "\n",
    "for i in range(len(b)):\n",
    "\n",
    "    model_imp_t = 0\n",
    "\n",
    "    for key in b[i].keys():\n",
    "\n",
    "        model_imp_t += (a[i][key][0] - b[i][key][0])/a[i][key][0]\n",
    "\n",
    "    imp_result.append(model_imp_t/len(b[i].keys()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.049357456497554374,\n",
       " 0.02075371552618405,\n",
       " -0.10713104463638444,\n",
       " -0.03307684376015278]"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Pytorch_1_11_20230812",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}