week12_lab3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "outputs": [
    {
     "data": {
      "text/plain": "      label  1x1  1x2  1x3  1x4  1x5  1x6  1x7  1x8  1x9  ...  28x19  28x20  \\\n0         7    0    0    0    0    0    0    0    0    0  ...      0      0   \n1         2    0    0    0    0    0    0    0    0    0  ...      0      0   \n2         1    0    0    0    0    0    0    0    0    0  ...      0      0   \n3         0    0    0    0    0    0    0    0    0    0  ...      0      0   \n4         4    0    0    0    0    0    0    0    0    0  ...      0      0   \n...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...    ...    ...   \n9995      2    0    0    0    0    0    0    0    0    0  ...      0      0   \n9996      3    0    0    0    0    0    0    0    0    0  ...      0      0   \n9997      4    0    0    0    0    0    0    0    0    0  ...      0      0   \n9998      5    0    0    0    0    0    0    0    0    0  ...      0      0   \n9999      6    0    0    0    0    0    0    0    0    0  ...      0      0   \n\n      28x21  28x22  28x23  28x24  28x25  28x26  28x27  28x28  \n0         0      0      0      0      0      0      0      0  \n1         0      0      0      0      0      0      0      0  \n2         0      0      0      0      0      0      0      0  \n3         0      0      0      0      0      0      0      0  \n4         0      0      0      0      0      0      0      0  \n...     ...    ...    ...    ...    ...    ...    ...    ...  \n9995      0      0      0      0      0      0      0      0  \n9996      0      0      0      0      0      0      0      0  \n9997      0      0      0      0      0      0      0      0  \n9998      0      0      0      0      0      0      0      0  \n9999      0      0      0      0      0      0      0      0  \n\n[10000 rows x 785 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>label</th>\n      <th>1x1</th>\n      <th>1x2</th>\n      <th>1x3</th>\n      <th>1x4</th>\n      <th>1x5</th>\n      <th>1x6</th>\n      <th>1x7</th>\n      <th>1x8</th>\n      <th>1x9</th>\n      <th>...</th>\n      <th>28x19</th>\n      <th>28x20</th>\n      <th>28x21</th>\n      <th>28x22</th>\n      <th>28x23</th>\n      <th>28x24</th>\n      <th>28x25</th>\n      <th>28x26</th>\n      <th>28x27</th>\n      <th>28x28</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>9995</th>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9996</th>\n      <td>3</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9997</th>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9998</th>\n      <td>5</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9999</th>\n      <td>6</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>10000 rows × 785 columns</p>\n</div>"
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/mnist_test.csv\")\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "outputs": [
    {
     "data": {
      "text/plain": "label    0\n1x1      0\n1x2      0\n1x3      0\n1x4      0\n        ..\n28x24    0\n28x25    0\n28x26    0\n28x27    0\n28x28    0\nLength: 785, dtype: int64"
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check dirty data\n",
    "df.isnull().sum()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "outputs": [],
   "source": [
    "# Split the dataset form target\n",
    "x, y = df.drop(['label'], axis=1), df['label']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "outputs": [
    {
     "data": {
      "text/plain": "      1x1  1x2  1x3  1x4  1x5  1x6  1x7  1x8  1x9  1x10  ...  28x19  28x20  \\\n0       0    0    0    0    0    0    0    0    0     0  ...      0      0   \n1       0    0    0    0    0    0    0    0    0     0  ...      0      0   \n2       0    0    0    0    0    0    0    0    0     0  ...      0      0   \n3       0    0    0    0    0    0    0    0    0     0  ...      0      0   \n4       0    0    0    0    0    0    0    0    0     0  ...      0      0   \n...   ...  ...  ...  ...  ...  ...  ...  ...  ...   ...  ...    ...    ...   \n9995    0    0    0    0    0    0    0    0    0     0  ...      0      0   \n9996    0    0    0    0    0    0    0    0    0     0  ...      0      0   \n9997    0    0    0    0    0    0    0    0    0     0  ...      0      0   \n9998    0    0    0    0    0    0    0    0    0     0  ...      0      0   \n9999    0    0    0    0    0    0    0    0    0     0  ...      0      0   \n\n      28x21  28x22  28x23  28x24  28x25  28x26  28x27  28x28  \n0         0      0      0      0      0      0      0      0  \n1         0      0      0      0      0      0      0      0  \n2         0      0      0      0      0      0      0      0  \n3         0      0      0      0      0      0      0      0  \n4         0      0      0      0      0      0      0      0  \n...     ...    ...    ...    ...    ...    ...    ...    ...  \n9995      0      0      0      0      0      0      0      0  \n9996      0      0      0      0      0      0      0      0  \n9997      0      0      0      0      0      0      0      0  \n9998      0      0      0      0      0      0      0      0  \n9999      0      0      0      0      0      0      0      0  \n\n[10000 rows x 784 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>1x1</th>\n      <th>1x2</th>\n      <th>1x3</th>\n      <th>1x4</th>\n      <th>1x5</th>\n      <th>1x6</th>\n      <th>1x7</th>\n      <th>1x8</th>\n      <th>1x9</th>\n      <th>1x10</th>\n      <th>...</th>\n      <th>28x19</th>\n      <th>28x20</th>\n      <th>28x21</th>\n      <th>28x22</th>\n      <th>28x23</th>\n      <th>28x24</th>\n      <th>28x25</th>\n      <th>28x26</th>\n      <th>28x27</th>\n      <th>28x28</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>9995</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9996</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9997</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9998</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9999</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>10000 rows × 784 columns</p>\n</div>"
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "outputs": [
    {
     "data": {
      "text/plain": "0       7\n1       2\n2       1\n3       0\n4       4\n       ..\n9995    2\n9996    3\n9997    4\n9998    5\n9999    6\nName: label, Length: 10000, dtype: int64"
     },
     "execution_count": 267,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.],\n       ...,\n       [0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.]])"
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scaling using standardScalr\n",
    "scaler = StandardScaler()\n",
    "scaler.fit_transform(x)\n",
    "x=scaler.transform(x)\n",
    "\n",
    "x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "outputs": [],
   "source": [
    "# Split the dataset into 5 subsets of equal size\n",
    "kf5 = KFold(n_splits=5, shuffle = True, random_state=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "outputs": [],
   "source": [
    "def gridSearch(dataX, dataY, k):\n",
    "    i = 1\n",
    "    for train, test in kf5.split(dataX):\n",
    "        print(\"Split \" + str(i))\n",
    "\n",
    "        x_train, x_test = dataX[train], dataX[test]\n",
    "        y_train, y_test = dataY[train], dataY[test]\n",
    "\n",
    "        knnCV = KNeighborsClassifier(n_neighbors=k)\n",
    "\n",
    "        scores = cross_val_score(knnCV, x_train, y_train, cv=5)\n",
    "        print(\"Case : initial K = 5\")\n",
    "        print(scores)\n",
    "        print(\"scores mean : {}\".format(np.mean(scores)))\n",
    "\n",
    "        knn = KNeighborsClassifier()\n",
    "\n",
    "        parameter = {'n_neighbors': np.arange(1, 25)}\n",
    "\n",
    "        knnGS = GridSearchCV(knn, param_grid=parameter, cv=5)\n",
    "\n",
    "        knnGS.fit(x_train, y_train)\n",
    "\n",
    "        print(\"Best Hyperparameter computing by GridSearch\")\n",
    "        print(knnGS.best_params_)\n",
    "        print(knnGS.best_score_)\n",
    "        print('\\n')\n",
    "\n",
    "        i = i + 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "outputs": [],
   "source": [
    "def randomGridSearch(dataX, dataY, k):\n",
    "    i = 1\n",
    "    for train, test in kf5.split(dataX):\n",
    "        print(\"Split \" + str(i))\n",
    "\n",
    "        x_train, x_test = dataX[train], dataX[test]\n",
    "        y_train, y_test = dataY[train], dataY[test]\n",
    "\n",
    "        knnCV = KNeighborsClassifier(n_neighbors=k)\n",
    "\n",
    "        scores = cross_val_score(knnCV, x_train, y_train, cv=5)\n",
    "        print(\"Case : initial K = 5\")\n",
    "        print(scores)\n",
    "        print(\"scores mean : {}\".format(np.mean(scores)))\n",
    "\n",
    "        parameter = {'n_neighbors': np.arange(1, 25)}\n",
    "\n",
    "        knn = KNeighborsClassifier()\n",
    "        knnRS = RandomizedSearchCV(knn, parameter, cv=5, scoring=\"accuracy\")\n",
    "\n",
    "        knnRS.fit(x_train, y_train)\n",
    "\n",
    "        print(\"Best Hyperparameter computing by Randomize GridSearch\")\n",
    "        print(knnRS.best_params_)\n",
    "        print(knnRS.best_score_)\n",
    "        print('\\n')\n",
    "\n",
    "        i = i + 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Case : K = 3, GridSearch\n",
      "\n",
      "Split 1\n",
      "Case : initial K = 5\n",
      "[0.8625   0.896875 0.896875 0.919375 0.9175  ]\n",
      "scores mean : 0.898625\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.898625\n",
      "\n",
      "\n",
      "Split 2\n",
      "Case : initial K = 5\n",
      "[0.870625 0.8875   0.901875 0.924375 0.9275  ]\n",
      "scores mean : 0.9023749999999999\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9023749999999999\n",
      "\n",
      "\n",
      "Split 3\n",
      "Case : initial K = 5\n",
      "[0.869375 0.89125  0.89875  0.9275   0.909375]\n",
      "scores mean : 0.89925\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 4}\n",
      "0.899625\n",
      "\n",
      "\n",
      "Split 4\n",
      "Case : initial K = 5\n",
      "[0.869375 0.8875   0.898125 0.91875  0.92625 ]\n",
      "scores mean : 0.9\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 4}\n",
      "0.90025\n",
      "\n",
      "\n",
      "Split 5\n",
      "Case : initial K = 5\n",
      "[0.8725   0.890625 0.901875 0.930625 0.920625]\n",
      "scores mean : 0.9032500000000001\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9032500000000001\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# k=3, GridSearch\n",
    "print(\"Case : K = 3, GridSearch\\n\")\n",
    "gridSearch(x, y, 3)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Case : K = 5, GridSearch\n",
      "\n",
      "Split 1\n",
      "Case : initial K = 5\n",
      "[0.86     0.884375 0.899375 0.919375 0.911875]\n",
      "scores mean : 0.8949999999999999\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.898625\n",
      "\n",
      "\n",
      "Split 2\n",
      "Case : initial K = 5\n",
      "[0.87     0.886875 0.89625  0.92875  0.920625]\n",
      "scores mean : 0.9005000000000001\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9023749999999999\n",
      "\n",
      "\n",
      "Split 3\n",
      "Case : initial K = 5\n",
      "[0.86625  0.889375 0.89625  0.93     0.914375]\n",
      "scores mean : 0.89925\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 4}\n",
      "0.899625\n",
      "\n",
      "\n",
      "Split 4\n",
      "Case : initial K = 5\n",
      "[0.871875 0.879375 0.896875 0.93     0.921875]\n",
      "scores mean : 0.9\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 4}\n",
      "0.90025\n",
      "\n",
      "\n",
      "Split 5\n",
      "Case : initial K = 5\n",
      "[0.86875 0.88875 0.9     0.9325  0.915  ]\n",
      "scores mean : 0.9010000000000001\n",
      "Best Hyperparameter computing by GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9032500000000001\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# k=5, GridSearch\n",
    "print(\"Case : K = 5, GridSearch\\n\")\n",
    "gridSearch(x, y, 5)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Case : K = 3, Randomized GridSearch\n",
      "\n",
      "Split 1\n",
      "Case : initial K = 5\n",
      "[0.8625   0.896875 0.896875 0.919375 0.9175  ]\n",
      "scores mean : 0.898625\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 6}\n",
      "0.895875\n",
      "\n",
      "\n",
      "Split 2\n",
      "Case : initial K = 5\n",
      "[0.870625 0.8875   0.901875 0.924375 0.9275  ]\n",
      "scores mean : 0.9023749999999999\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 5}\n",
      "0.9005000000000001\n",
      "\n",
      "\n",
      "Split 3\n",
      "Case : initial K = 5\n",
      "[0.869375 0.89125  0.89875  0.9275   0.909375]\n",
      "scores mean : 0.89925\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 5}\n",
      "0.89925\n",
      "\n",
      "\n",
      "Split 4\n",
      "Case : initial K = 5\n",
      "[0.869375 0.8875   0.898125 0.91875  0.92625 ]\n",
      "scores mean : 0.9\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9\n",
      "\n",
      "\n",
      "Split 5\n",
      "Case : initial K = 5\n",
      "[0.8725   0.890625 0.901875 0.930625 0.920625]\n",
      "scores mean : 0.9032500000000001\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9032500000000001\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# k=3 Randomized GridSearch\n",
    "print(\"Case : K = 3, Randomized GridSearch\\n\")\n",
    "randomGridSearch(x, y, 3)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Case : K = 5, Randomized GridSearch\n",
      "\n",
      "Split 1\n",
      "Case : initial K = 5\n",
      "[0.86     0.884375 0.899375 0.919375 0.911875]\n",
      "scores mean : 0.8949999999999999\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 4}\n",
      "0.8952500000000001\n",
      "\n",
      "\n",
      "Split 2\n",
      "Case : initial K = 5\n",
      "[0.87     0.886875 0.89625  0.92875  0.920625]\n",
      "scores mean : 0.9005000000000001\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9023749999999999\n",
      "\n",
      "\n",
      "Split 3\n",
      "Case : initial K = 5\n",
      "[0.86625  0.889375 0.89625  0.93     0.914375]\n",
      "scores mean : 0.89925\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.89925\n",
      "\n",
      "\n",
      "Split 4\n",
      "Case : initial K = 5\n",
      "[0.871875 0.879375 0.896875 0.93     0.921875]\n",
      "scores mean : 0.9\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 6}\n",
      "0.9001250000000001\n",
      "\n",
      "\n",
      "Split 5\n",
      "Case : initial K = 5\n",
      "[0.86875 0.88875 0.9     0.9325  0.915  ]\n",
      "scores mean : 0.9010000000000001\n",
      "Best Hyperparameter computing by Randomize GridSearch\n",
      "{'n_neighbors': 3}\n",
      "0.9032500000000001\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# k=5 Randomized GridSearch\n",
    "print(\"Case : K = 5, Randomized GridSearch\\n\")\n",
    "randomGridSearch(x, y, 5)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}