06_Random_Cut_Forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sagemaker.amazon.amazon_estimator import get_image_uri\n",
    "import boto3\n",
    "import re\n",
    "import sagemaker\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "pd.options.display.max_rows = 100\n",
    "pd.options.display.float_format = '{:.3f}'.format\n",
    "pd.options.display.max_columns = 100\n",
    "\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "#Manage interactions with the Amazon SageMaker APIs and any other AWS services needed.\n",
    "#manipulating entities and resources that Amazon SageMaker uses, such as training jobs, endpoints, and input datasets in S3.\n",
    "sess = sagemaker.Session()\n",
    "bucket = sess.default_bucket()\n",
    "prefix = 'DEMO-random-cut-forest'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.25 s, sys: 94.6 ms, total: 2.34 s\n",
      "Wall time: 2.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import pandas as pd\n",
    "import urllib.request\n",
    "\n",
    "data_filename = './data/creditcard.csv'\n",
    "\n",
    "\n",
    "credit_df = pd.read_csv(data_filename, delimiter=',')\n",
    "\n",
    "\n",
    "inliers = credit_df[credit_df.Class==0]\n",
    "inliers = inliers.drop(['Class'], axis=1)\n",
    "\n",
    "outliers = credit_df[credit_df.Class==1]\n",
    "outliers_class = credit_df[credit_df.Class==1]['Class']\n",
    "outliers = outliers.drop(['Class'], axis=1)\n",
    "\n",
    "credit_df = credit_df.drop(['Class'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Store Data on S3\n",
    "The Random Cut Forest Algorithm accepts data in RecordIO Protobuf format. The SageMaker Python API provides helper functions for easily converting your data into this format. Below we convert the temperature sensor data and upload it to the bucket + prefix Amazon S3 destination specified at the beginning of this notebook in the Setup AWS Credentials section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_s3_file_csv(bucket, file_data, dataset, header=None, index=None):\n",
    "    csv_buffer = io.StringIO()\n",
    "    dataset.to_csv(csv_buffer, header=header, index=index)\n",
    "    s3_client = boto3.client('s3')\n",
    "    response = s3_client.put_object(Bucket=bucket, Key=file_data, Body=csv_buffer.getvalue())\n",
    "    return response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the training job and creating the estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker import RandomCutForest\n",
    "import sagemaker\n",
    "import os, io\n",
    "\n",
    "\n",
    "train_channel = os.path.join(prefix, 'train')\n",
    "\n",
    "feature_dim = credit_df.shape[1]\n",
    "\n",
    "# Save data at training folder\n",
    "response = save_s3_file_csv(bucket, prefix, inliers, \n",
    "                 header=None, \n",
    "                 index=None)\n",
    "\n",
    "\n",
    "rcf = RandomCutForest(role=role,\n",
    "                      train_instance_count=1,\n",
    "                      train_instance_type='ml.m4.xlarge',\n",
    "                      data_location='s3://{}/{}/train_rcf/'.format(bucket, prefix), # Folder with '/'\n",
    "                      output_path='s3://{}/{}/output'.format(bucket, prefix),\n",
    "                      num_samples_per_tree=300,\n",
    "                      num_trees=200) # Min number of tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start the training process on Amazon SageMaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-10-21 10:19:54 Starting - Starting the training job...\n",
      "2019-10-21 10:19:59 Starting - Launching requested ML instances.........\n",
      "2019-10-21 10:21:31 Starting - Preparing the instances for training......\n",
      "2019-10-21 10:22:29 Downloading - Downloading input data...\n",
      "2019-10-21 10:23:24 Training - Training image download completed. Training in progress..\u001b[31mDocker entrypoint called with argument(s): train\u001b[0m\n",
      "\u001b[31m/opt/amazon/lib/python2.7/site-packages/scipy/_lib/_numpy_compat.py:10: DeprecationWarning: Importing from numpy.testing.nosetester is deprecated, import from numpy.testing instead.\n",
      "  from numpy.testing.nosetester import import_nose\u001b[0m\n",
      "\u001b[31m/opt/amazon/lib/python2.7/site-packages/scipy/stats/morestats.py:12: DeprecationWarning: Importing from numpy.testing.decorators is deprecated, import from numpy.testing instead.\n",
      "  from numpy.testing.decorators import setastest\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Reading default configuration from /opt/amazon/lib/python2.7/site-packages/algorithm/resources/default-conf.json: {u'_ftp_port': 8999, u'num_samples_per_tree': 256, u'_tuning_objective_metric': u'', u'_num_gpus': u'auto', u'_log_level': u'info', u'_kvstore': u'dist_async', u'force_dense': u'true', u'epochs': 1, u'num_trees': 100, u'eval_metrics': [u'accuracy', u'precision_recall_fscore'], u'_num_kv_servers': u'auto', u'mini_batch_size': 1000}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Reading provided configuration from /opt/ml/input/config/hyperparameters.json: {u'mini_batch_size': u'1000', u'feature_dim': u'30', u'num_samples_per_tree': u'300', u'num_trees': u'200'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Final configuration: {u'_ftp_port': 8999, u'num_samples_per_tree': u'300', u'_tuning_objective_metric': u'', u'_num_gpus': u'auto', u'_log_level': u'info', u'_kvstore': u'dist_async', u'force_dense': u'true', u'epochs': 1, u'feature_dim': u'30', u'num_trees': u'200', u'eval_metrics': [u'accuracy', u'precision_recall_fscore'], u'_num_kv_servers': u'auto', u'mini_batch_size': u'1000'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 WARNING 140115100460864] Loggers have already been setup.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Launching parameter server for role scheduler\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] {'ECS_CONTAINER_METADATA_URI': 'http://169.254.170.2/v3/3953f617-c45b-48fa-ab0f-520ed6cba8d1', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION': '2', 'PATH': '/opt/amazon/bin:/usr/local/nvidia/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/amazon/bin:/opt/amazon/bin', 'SAGEMAKER_HTTP_PORT': '8080', 'HOME': '/root', 'PYTHONUNBUFFERED': 'TRUE', 'CANONICAL_ENVROOT': '/opt/amazon', 'LD_LIBRARY_PATH': '/opt/amazon/lib/python2.7/site-packages/cv2/../../../../lib:/usr/local/nvidia/lib64:/opt/amazon/lib', 'MXNET_KVSTORE_BIGARRAY_BOUND': '400000000', 'LANG': 'en_US.utf8', 'DMLC_INTERFACE': 'eth0', 'SHLVL': '1', 'AWS_REGION': 'us-east-1', 'NVIDIA_VISIBLE_DEVICES': 'void', 'TRAINING_JOB_NAME': 'randomcutforest-2019-10-21-10-19-54-591', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'cpp', 'ENVROOT': '/opt/amazon', 'SAGEMAKER_DATA_PATH': '/opt/ml', 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 'NVIDIA_REQUIRE_CUDA': 'cuda>=9.0', 'OMP_NUM_THREADS': '2', 'HOSTNAME': 'ip-10-0-221-114.ec2.internal', 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI': '/v2/credentials/14a4540a-ac3b-43dc-bf15-0cddffe01c98', 'PWD': '/', 'TRAINING_JOB_ARN': 'arn:aws:sagemaker:us-east-1:625616379791:training-job/randomcutforest-2019-10-21-10-19-54-591', 'AWS_EXECUTION_ENV': 'AWS_ECS_EC2'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] envs={'ECS_CONTAINER_METADATA_URI': 'http://169.254.170.2/v3/3953f617-c45b-48fa-ab0f-520ed6cba8d1', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION': '2', 'DMLC_NUM_WORKER': '1', 'DMLC_PS_ROOT_PORT': '9000', 'PATH': '/opt/amazon/bin:/usr/local/nvidia/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/amazon/bin:/opt/amazon/bin', 'SAGEMAKER_HTTP_PORT': '8080', 'HOME': '/root', 'PYTHONUNBUFFERED': 'TRUE', 'CANONICAL_ENVROOT': '/opt/amazon', 'LD_LIBRARY_PATH': '/opt/amazon/lib/python2.7/site-packages/cv2/../../../../lib:/usr/local/nvidia/lib64:/opt/amazon/lib', 'MXNET_KVSTORE_BIGARRAY_BOUND': '400000000', 'LANG': 'en_US.utf8', 'DMLC_INTERFACE': 'eth0', 'SHLVL': '1', 'DMLC_PS_ROOT_URI': '10.0.221.114', 'AWS_REGION': 'us-east-1', 'NVIDIA_VISIBLE_DEVICES': 'void', 'TRAINING_JOB_NAME': 'randomcutforest-2019-10-21-10-19-54-591', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'cpp', 'ENVROOT': '/opt/amazon', 'SAGEMAKER_DATA_PATH': '/opt/ml', 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 'NVIDIA_REQUIRE_CUDA': 'cuda>=9.0', 'OMP_NUM_THREADS': '2', 'HOSTNAME': 'ip-10-0-221-114.ec2.internal', 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI': '/v2/credentials/14a4540a-ac3b-43dc-bf15-0cddffe01c98', 'DMLC_ROLE': 'scheduler', 'PWD': '/', 'DMLC_NUM_SERVER': '1', 'TRAINING_JOB_ARN': 'arn:aws:sagemaker:us-east-1:625616379791:training-job/randomcutforest-2019-10-21-10-19-54-591', 'AWS_EXECUTION_ENV': 'AWS_ECS_EC2'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Launching parameter server for role server\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] {'ECS_CONTAINER_METADATA_URI': 'http://169.254.170.2/v3/3953f617-c45b-48fa-ab0f-520ed6cba8d1', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION': '2', 'PATH': '/opt/amazon/bin:/usr/local/nvidia/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/amazon/bin:/opt/amazon/bin', 'SAGEMAKER_HTTP_PORT': '8080', 'HOME': '/root', 'PYTHONUNBUFFERED': 'TRUE', 'CANONICAL_ENVROOT': '/opt/amazon', 'LD_LIBRARY_PATH': '/opt/amazon/lib/python2.7/site-packages/cv2/../../../../lib:/usr/local/nvidia/lib64:/opt/amazon/lib', 'MXNET_KVSTORE_BIGARRAY_BOUND': '400000000', 'LANG': 'en_US.utf8', 'DMLC_INTERFACE': 'eth0', 'SHLVL': '1', 'AWS_REGION': 'us-east-1', 'NVIDIA_VISIBLE_DEVICES': 'void', 'TRAINING_JOB_NAME': 'randomcutforest-2019-10-21-10-19-54-591', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'cpp', 'ENVROOT': '/opt/amazon', 'SAGEMAKER_DATA_PATH': '/opt/ml', 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 'NVIDIA_REQUIRE_CUDA': 'cuda>=9.0', 'OMP_NUM_THREADS': '2', 'HOSTNAME': 'ip-10-0-221-114.ec2.internal', 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI': '/v2/credentials/14a4540a-ac3b-43dc-bf15-0cddffe01c98', 'PWD': '/', 'TRAINING_JOB_ARN': 'arn:aws:sagemaker:us-east-1:625616379791:training-job/randomcutforest-2019-10-21-10-19-54-591', 'AWS_EXECUTION_ENV': 'AWS_ECS_EC2'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] envs={'ECS_CONTAINER_METADATA_URI': 'http://169.254.170.2/v3/3953f617-c45b-48fa-ab0f-520ed6cba8d1', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION': '2', 'DMLC_NUM_WORKER': '1', 'DMLC_PS_ROOT_PORT': '9000', 'PATH': '/opt/amazon/bin:/usr/local/nvidia/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/amazon/bin:/opt/amazon/bin', 'SAGEMAKER_HTTP_PORT': '8080', 'HOME': '/root', 'PYTHONUNBUFFERED': 'TRUE', 'CANONICAL_ENVROOT': '/opt/amazon', 'LD_LIBRARY_PATH': '/opt/amazon/lib/python2.7/site-packages/cv2/../../../../lib:/usr/local/nvidia/lib64:/opt/amazon/lib', 'MXNET_KVSTORE_BIGARRAY_BOUND': '400000000', 'LANG': 'en_US.utf8', 'DMLC_INTERFACE': 'eth0', 'SHLVL': '1', 'DMLC_PS_ROOT_URI': '10.0.221.114', 'AWS_REGION': 'us-east-1', 'NVIDIA_VISIBLE_DEVICES': 'void', 'TRAINING_JOB_NAME': 'randomcutforest-2019-10-21-10-19-54-591', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'cpp', 'ENVROOT': '/opt/amazon', 'SAGEMAKER_DATA_PATH': '/opt/ml', 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 'NVIDIA_REQUIRE_CUDA': 'cuda>=9.0', 'OMP_NUM_THREADS': '2', 'HOSTNAME': 'ip-10-0-221-114.ec2.internal', 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI': '/v2/credentials/14a4540a-ac3b-43dc-bf15-0cddffe01c98', 'DMLC_ROLE': 'server', 'PWD': '/', 'DMLC_NUM_SERVER': '1', 'TRAINING_JOB_ARN': 'arn:aws:sagemaker:us-east-1:625616379791:training-job/randomcutforest-2019-10-21-10-19-54-591', 'AWS_EXECUTION_ENV': 'AWS_ECS_EC2'}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Environment: {'ECS_CONTAINER_METADATA_URI': 'http://169.254.170.2/v3/3953f617-c45b-48fa-ab0f-520ed6cba8d1', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION': '2', 'DMLC_PS_ROOT_PORT': '9000', 'DMLC_NUM_WORKER': '1', 'SAGEMAKER_HTTP_PORT': '8080', 'PATH': '/opt/amazon/bin:/usr/local/nvidia/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/amazon/bin:/opt/amazon/bin', 'PYTHONUNBUFFERED': 'TRUE', 'CANONICAL_ENVROOT': '/opt/amazon', 'LD_LIBRARY_PATH': '/opt/amazon/lib/python2.7/site-packages/cv2/../../../../lib:/usr/local/nvidia/lib64:/opt/amazon/lib', 'MXNET_KVSTORE_BIGARRAY_BOUND': '400000000', 'LANG': 'en_US.utf8', 'DMLC_INTERFACE': 'eth0', 'SHLVL': '1', 'DMLC_PS_ROOT_URI': '10.0.221.114', 'AWS_REGION': 'us-east-1', 'NVIDIA_VISIBLE_DEVICES': 'void', 'TRAINING_JOB_NAME': 'randomcutforest-2019-10-21-10-19-54-591', 'HOME': '/root', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'cpp', 'ENVROOT': '/opt/amazon', 'SAGEMAKER_DATA_PATH': '/opt/ml', 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 'NVIDIA_REQUIRE_CUDA': 'cuda>=9.0', 'OMP_NUM_THREADS': '2', 'HOSTNAME': 'ip-10-0-221-114.ec2.internal', 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI': '/v2/credentials/14a4540a-ac3b-43dc-bf15-0cddffe01c98', 'DMLC_ROLE': 'worker', 'PWD': '/', 'DMLC_NUM_SERVER': '1', 'TRAINING_JOB_ARN': 'arn:aws:sagemaker:us-east-1:625616379791:training-job/randomcutforest-2019-10-21-10-19-54-591', 'AWS_EXECUTION_ENV': 'AWS_ECS_EC2'}\u001b[0m\n",
      "\u001b[31mProcess 33 is a shell:scheduler.\u001b[0m\n",
      "\u001b[31mProcess 34 is a shell:server.\u001b[0m\n",
      "\u001b[31mProcess 1 is a worker.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Using default worker.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Loaded iterator creator application/x-recordio-protobuf for content type ('application/x-recordio-protobuf', '1.0')\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Verifying hyperparamemters...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Hyperparameters are correct.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Validating that feature_dim agrees with dimensions in training data...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] feature_dim is correct.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Validating memory limits...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Available memory in bytes: 15398207488\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Estimated sample size in bytes: 28800000\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Estimated memory needed to build the forest in bytes: 72000000\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Memory limits validated.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Starting cluster sharing facilities...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140115100460864] Create Store: dist_async\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:28] >>> starting FTP server on 0.0.0.0:8999, pid=1 <<<\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140113487034112] >>> starting FTP server on 0.0.0.0:8999, pid=1 <<<\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:28] poller: <class 'pyftpdlib.ioloop.Epoll'>\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140113487034112] poller: <class 'pyftpdlib.ioloop.Epoll'>\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:28] masquerade (NAT) address: None\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140113487034112] masquerade (NAT) address: None\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140113487034112] passive ports: None\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:28] passive ports: None\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:28] use sendfile(2): False\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:28 INFO 140113487034112] use sendfile(2): False\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Cluster sharing facilities started.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Verifying all workers are accessible...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] All workers accessible.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Initializing Sampler...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Sampler correctly initialized.\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"initialize.time\": {\"count\": 1, \"max\": 779.7510623931885, \"sum\": 779.7510623931885, \"min\": 779.7510623931885}}, \"EndTime\": 1571653409.083034, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653408.296341}\n",
      "\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Total Batches Seen\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Total Records Seen\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Reset Count\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}}, \"EndTime\": 1571653409.08328, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"init_train_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653409.083216}\n",
      "\u001b[0m\n",
      "\u001b[31m[2019-10-21 10:23:29.085] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 0, \"duration\": 787, \"num_examples\": 1, \"num_bytes\": 268000}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Sampling training data...\u001b[0m\n",
      "\u001b[31m[2019-10-21 10:23:29.688] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 1, \"duration\": 602, \"num_examples\": 285, \"num_bytes\": 76196420}\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Sampling training data completed.\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"epochs\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"update.time\": {\"count\": 1, \"max\": 646.5420722961426, \"sum\": 646.5420722961426, \"min\": 646.5420722961426}}, \"EndTime\": 1571653409.731699, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653409.083148}\n",
      "\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Early stop condition met. Stopping training.\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] #progress_metric: host=algo-1, completed 100 % epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 285, \"sum\": 285.0, \"min\": 285}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 285, \"sum\": 285.0, \"min\": 285}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 284315, \"sum\": 284315.0, \"min\": 284315}, \"Total Batches Seen\": {\"count\": 1, \"max\": 285, \"sum\": 285.0, \"min\": 285}, \"Total Records Seen\": {\"count\": 1, \"max\": 284315, \"sum\": 284315.0, \"min\": 284315}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 284315, \"sum\": 284315.0, \"min\": 284315}, \"Reset Count\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}}, \"EndTime\": 1571653409.732075, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\", \"epoch\": 0}, \"StartTime\": 1571653409.085118}\n",
      "\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] #throughput_metric: host=algo-1, train throughput=439367.159073 records/second\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Master node: building Random Cut Forest...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Gathering samples...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] 60000 samples gathered\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Building Random Cut Forest...\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:29 INFO 140115100460864] Random Cut Forest built: \n",
      "\u001b[0m\n",
      "\u001b[31mForestInfo{num_trees: 200, num_samples_in_forest: 60000, num_samples_per_tree: 300, sample_dim: 30, shingle_size: 1, trees_num_nodes: [573, 563, 561, 579, 565, 583, 567, 575, 565, 569, 571, 571, 571, 563, 571, 561, 573, 573, 567, 565, 579, 555, 571, 567, 565, 563, 565, 575, 573, 561, 555, 565, 565, 567, 575, 555, 571, 575, 583, 567, 567, 559, 575, 559, 565, 571, 557, 563, 571, 569, 581, 565, 565, 565, 585, 571, 571, 571, 565, 565, 555, 563, 569, 573, 577, 561, 561, 561, 569, 569, 587, 563, 559, 557, 559, 565, 569, 577, 551, 561, 569, 553, 567, 559, 563, 571, 575, 577, 561, 567, 565, 563, 565, 565, 563, 569, 581, 557, 563, 563, 573, 573, 579, 557, 571, 571, 571, 571, 565, 573, 561, 585, 559, 581, 573, 565, 561, 581, 573, 565, 557, 567, 555, 571, 577, 573, 565, 561, 571, 551, 567, 563, 569, 565, 577, 567, 579, 573, 567, 563, 579, 559, 571, 577, 569, 561, 571, 563, 581, 569, 561, 573, 577, 565, 569, 567, 571, 561, 567, 557, 575, 557, 569, 567, 575, 577, 561, 565, 565, 571, 569, 569, 579, 563, 567, 561, 567, 563, 557, 567, 569, 577, 577, 563, 561, 573, 567, 573, 567, 567, 565, 571, 563, 565, 545, 565, 569, 575, 569, 555, ], trees_depth: [20, 18, 18, 20, 19, 19, 17, 19, 15, 18, 19, 18, 17, 20, 18, 18, 21, 18, 17, 20, 18, 16, 20, 19, 17, 19, 22, 20, 17, 18, 17, 16, 21, 20, 21, 20, 16, 20, 18, 19, 15, 21, 21, 17, 20, 24, 18, 22, 19, 19, 22, 19, 18, 16, 21, 21, 22, 18, 22, 22, 22, 17, 23, 23, 16, 21, 20, 19, 21, 19, 21, 20, 16, 19, 22, 19, 21, 18, 18, 20, 19, 20, 18, 18, 22, 18, 19, 20, 20, 17, 24, 17, 21, 22, 19, 20, 17, 21, 18, 19, 18, 17, 21, 18, 18, 22, 18, 17, 19, 18, 22, 18, 17, 22, 18, 17, 21, 19, 17, 21, 15, 18, 19, 22, 18, 17, 20, 17, 20, 20, 17, 19, 19, 19, 21, 19, 20, 21, 17, 17, 20, 18, 22, 16, 17, 20, 20, 18, 19, 19, 19, 18, 19, 20, 17, 19, 20, 20, 18, 20, 17, 19, 19, 17, 17, 18, 23, 18, 22, 19, 22, 17, 22, 19, 27, 18, 19, 16, 18, 18, 20, 17, 19, 18, 17, 18, 17, 17, 18, 20, 21, 17, 18, 19, 21, 21, 19, 16, 18, 17, ], max_num_nodes: 587, min_num_nodes: 545, avg_num_nodes: 567, max_tree_depth: 27, min_tree_depth: 15, avg_tree_depth: 19, mem_size: 64477600}\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"finalize.time\": {\"count\": 1, \"max\": 235.93807220458984, \"sum\": 235.93807220458984, \"min\": 235.93807220458984}, \"model.bytes\": {\"count\": 1, \"max\": 64477600, \"sum\": 64477600.0, \"min\": 64477600}, \"fit_model.time\": {\"count\": 1, \"max\": 162.1878147125244, \"sum\": 162.1878147125244, \"min\": 162.1878147125244}}, \"EndTime\": 1571653409.968323, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653409.731799}\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[10/21/2019 10:23:31 INFO 140115100460864] Master node: Serializing the RandomCutForest model\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"serialize_model.time\": {\"count\": 1, \"max\": 1633.1660747528076, \"sum\": 1633.1660747528076, \"min\": 1633.1660747528076}}, \"EndTime\": 1571653411.601644, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653409.968425}\n",
      "\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:31 INFO 140115100460864] Test data is not provided.\u001b[0m\n",
      "\u001b[31m[I 19-10-21 10:23:31] >>> shutting down FTP server (0 active fds) <<<\u001b[0m\n",
      "\u001b[31m[10/21/2019 10:23:31 INFO 140113487034112] >>> shutting down FTP server (0 active fds) <<<\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"totaltime\": {\"count\": 1, \"max\": 3655.3308963775635, \"sum\": 3655.3308963775635, \"min\": 3655.3308963775635}, \"setuptime\": {\"count\": 1, \"max\": 212.69989013671875, \"sum\": 212.69989013671875, \"min\": 212.69989013671875}}, \"EndTime\": 1571653411.730252, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"RandomCutForest\"}, \"StartTime\": 1571653411.601723}\n",
      "\u001b[0m\n",
      "\n",
      "2019-10-21 10:23:35 Uploading - Uploading generated training model\n",
      "2019-10-21 10:24:01 Completed - Training job completed\n",
      "Training seconds: 92\n",
      "Billable seconds: 92\n"
     ]
    }
   ],
   "source": [
    "rcf.fit(rcf.record_set(inliers.values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploying the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------------------------------------------------!Deployment completed\n"
     ]
    }
   ],
   "source": [
    "predictor = rcf.deploy(initial_instance_count=1,\n",
    "                       instance_type='ml.t2.medium', \n",
    "                       wait=True)\n",
    "\n",
    "print('Deployment completed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.76\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.predictor import csv_serializer, json_deserializer\n",
    "\n",
    "predictor.content_type = 'text/csv'\n",
    "predictor.serializer = csv_serializer\n",
    "predictor.accept = 'application/json'\n",
    "predictor.deserializer = json_deserializer\n",
    "\n",
    "results = predictor.predict(outliers.values)\n",
    "scores = np.array([datum['score'] for datum in results['scores']])\n",
    "\n",
    "#Calculate the score cutoff threshold\n",
    "score_mean = scores.mean()\n",
    "score_std = scores.std()\n",
    "score_cutoff = 0.76\n",
    "\n",
    "print(score_cutoff)\n",
    "anomalies = scores[scores > score_cutoff]\n",
    "anomaly_indices = np.arange(len(scores))[scores > score_cutoff]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy in Detecting Legit Cases: 91.46341463414635 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy in Detecting Legit Cases:\", len(anomalies)/len(outliers) * 100, '%')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}