DEBUG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import json\n",
    "import os\n",
    "import random\n",
    "import subprocess\n",
    "from contextlib import redirect_stdout\n",
    "\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard\n",
    "from keras.layers import Conv2D, Conv2DTranspose, Input, LeakyReLU\n",
    "from keras.models import Model, load_model\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.utils.multi_gpu_utils import multi_gpu_model\n",
    "from matplotlib.colors import hsv_to_rgb, rgb_to_hsv\n",
    "from PIL import Image, ImageEnhance\n",
    "\n",
    "from sklearn import svm\n",
    "import tensorflow.io.gfile as gfile\n",
    "\n",
    "\n",
    "import trainer.parse_file_list\n",
    "from trainer.gpu_utils import ModelCkptMultiGPU, get_available_gpus\n",
    "from trainer.model_ckpt_gc import ModelCheckpointGC\n",
    "from trainer.plot_utils import plot_hist, plot_mvtec, plot_samples, savefig\n",
    "\n",
    "# from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard\n",
    "# from tensorflow.keras.layers import Conv2D, Conv2DTranspose, Input\n",
    "# from tensorflow.keras.models import Model, load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set random seed for redproducible results\n",
    "RAND = 42\n",
    "np.random.seed(RAND)\n",
    "tf.set_random_seed(RAND)\n",
    "random.seed(RAND)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from types import SimpleNamespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUCKET = \"gs://e4u-anomaly-detection\"\n",
    "args  = SimpleNamespace(use_case=\"spm\",\n",
    "                        datadir=\"gs://training-data-images/\",\n",
    "                       logdir=\"gs://e4u-anomaly-detection/logs/DEBUG_AD\",\n",
    "                       ckptdir=\"gs://e4u-anomaly-detection/ckpts/DEBUG_AD\",\n",
    "                       imgdir=\"gs://e4u-anomaly-detection/imgs/DEBUG_AD\",\n",
    "                       evaldir=\"gs://e4u-anomaly-detection/eval/DEBUG_AD\",\n",
    "                       epochs=50,\n",
    "                       batch_size=128,\n",
    "                       filters=[32,32,32,64,64,128,256,128,64],\n",
    "                       ldim=256)\n",
    "\n",
    "global USE_CASE\n",
    "USE_CASE = args.use_case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_image_file_names(dir_path):\n",
    "    \"\"\"Returns a list of absolute file paths for relative dir input with all relevant file names.\"\"\"\n",
    "    return [os.path.join(dir_path, p) for p in gfile.ListDirectory(dir_path)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_path_to_data(data_path, mode):\n",
    "    return os.path.join(data_path, USE_CASE, mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(data_path):\n",
    "    normal_path = os.path.join(data_path, USE_CASE, \"train\", \"good\")\n",
    "    normal_file_names = read_image_file_names(normal_path)\n",
    "    random.shuffle(normal_file_names)\n",
    "\n",
    "    train_validation_split = (len(normal_file_names) // 10) * 8  # equals .8\n",
    "    train_file_names, validation_file_names = normal_file_names[\n",
    "        :train_validation_split], normal_file_names[train_validation_split:]\n",
    "\n",
    "    test_path = os.path.join(data_path, USE_CASE, \"test\")\n",
    "\n",
    "    normal_test_file_names = read_image_file_names(\n",
    "        os.path.join(test_path, \"good\"))\n",
    "\n",
    "    anomaly_test_file_names = anomaly_train_file_names = []\n",
    "    # iterate of test list dir because anomalies are named dynamically\n",
    "    for i, p in enumerate(filter(lambda x: x != \"good\", gfile.ListDirectory(test_path))):\n",
    "        if i == 0:\n",
    "            anomaly_test_file_names += read_image_file_names(\n",
    "                os.path.join(test_path, p))\n",
    "        else:\n",
    "            anomaly_train_file_names += read_image_file_names(\n",
    "                os.path.join(test_path, p))\n",
    "\n",
    "    print(\"Splitting data:\\n%4d Normal Train Samples\\n%4d Normal Validation Samples\\n%4d Normal Test Samples\\n%4d Anomaly Train Samples\\n%4d Anomaly Test Samples\" % (\n",
    "        len(train_file_names), len(validation_file_names), len(normal_test_file_names), len(anomaly_train_file_names), len(anomaly_test_file_names)))\n",
    "\n",
    "    return train_file_names, validation_file_names, normal_test_file_names, anomaly_train_file_names, anomaly_test_file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.io.gfile\n",
    "import os\n",
    "from urllib.parse import urlparse\n",
    "import numpy as np\n",
    "import base64\n",
    "\n",
    "DATA_SET_FRAC = 50 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(data_path_normal, data_path_anomaly):\n",
    "    def split_bucket_parts(path):\n",
    "        url = urlparse(path)  # [:-1] cuts the \\n from the lines\n",
    "        bucket = \"%s://%s\" % (url.scheme, url.netloc)\n",
    "\n",
    "        # [1:] removes leading / to avoid empty string in dir_parts\n",
    "        dir_parts = os.path.normpath(url.path[1:]).split(os.sep)\n",
    "        \n",
    "        return bucket, dir_parts\n",
    "\n",
    "    def get_pipes(lines):\n",
    "        splitted_lines = np.array([split_bucket_parts(l)[1] for l in lines])\n",
    "        noc = list(set(splitted_lines[:, 2]))\n",
    "        return noc\n",
    "\n",
    "    def split_train_test(lines, train_pipes):\n",
    "\n",
    "        # find line without a train class in it\n",
    "        tts_lines = int(.8 * len(lines))-1\n",
    "        i = tts_lines\n",
    "        for l in lines[tts_lines:]:\n",
    "            if not any(tp in l for tp in train_pipes):\n",
    "                break\n",
    "            i += 1\n",
    "\n",
    "        train_file_paths = lines[:i]\n",
    "        test_file_paths = lines[i:]\n",
    "        # print(len(train_file_paths)/len(lines),\n",
    "        #   len(test_file_paths)/len(lines))\n",
    "        return train_file_paths, test_file_paths\n",
    "    # get normal data\n",
    "\n",
    "    with gfile.GFile(data_path_normal, \"r\") as f:\n",
    "        lines = f.readlines()\n",
    "        lines = lines[0::DATA_SET_FRAC]\n",
    "        lines = [l[:-1] for l in lines]  # remove trailing \\n char\n",
    "    bucket = split_bucket_parts(lines[0])[0]\n",
    "    pipes = get_pipes(lines) # pipe names e.g \"1-11111 1-11110\"\n",
    "    nop = len(pipes)  # number of pipes\n",
    "    \n",
    "    # 80% of the pipe sections should be used for train and validation, therefor pipe section names are split\n",
    "    train_val_pipes = pipes[:int(.8 * nop) ] # names of pipes which should be used for training and validation\n",
    "\n",
    "    normal_train_file_paths, normal_test_file_paths = split_train_test(\n",
    "        lines, train_val_pipes)\n",
    "    \n",
    "    train_pipes = pipes[:int(.8 * nop) ] # names of pipes which should be used for training and validation\n",
    "    normal_train_file_paths, normal_validation_file_paths = split_train_test(\n",
    "        lines[:len(normal_train_file_paths)], train_pipes)\n",
    "    \n",
    "    train_ratio = len(normal_train_file_paths)/len(lines)\n",
    "    val_ratio = len(normal_validation_file_paths)/len(lines)\n",
    "    test_ratio = len(normal_test_file_paths)/len(lines)\n",
    "\n",
    "    # get anomaly data\n",
    "    with gfile.GFile(data_path_anomaly, \"r\") as f:\n",
    "        lines = f.readlines()\n",
    "        lines = lines[0::(DATA_SET_FRAC//10)]\n",
    "        file_paths = [split_bucket_parts(l[:-1])[1][-1] for l in lines]\n",
    "\n",
    "    decoded_lines = []\n",
    "    for l in file_paths:\n",
    "        rest = os.path.join(*base64.b64decode(\n",
    "            l[:-4]).decode(\"utf-8\").split(os.sep)[-3:])+\".jpg\"\n",
    "        new_path = os.path.join(bucket,  \"out\", rest)\n",
    "        decoded_lines.append(new_path)\n",
    "\n",
    "    anomaly_train_file_paths, anomaly_test_file_paths = split_train_test(\n",
    "        decoded_lines, train_pipes)\n",
    "    \n",
    "    print(\"Splitting data:\\n%4d Normal Train Samples\\n%4d Normal Validation Samples\\n%4d Normal Test Samples\\n%4d Anomaly Train Samples\\n%4d Anomaly Test Samples\" % (\n",
    "        len(normal_train_file_paths), len(normal_validation_file_paths), len(normal_test_file_paths), len(anomaly_train_file_paths), len(anomaly_test_file_paths)))\n",
    "    print(\"Train-Val-Test Ratio: %.3f-%.3f-%.3f\"%(train_ratio,val_ratio,test_ratio))\n",
    "    return normal_train_file_paths, normal_validation_file_paths, normal_test_file_paths, anomaly_train_file_paths, anomaly_test_file_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://training-data-images/no_damage.txt\n",
      "Splitting data:\n",
      "5810 Normal Train Samples\n",
      "1232 Normal Validation Samples\n",
      "1685 Normal Test Samples\n",
      " 452 Anomaly Train Samples\n",
      "  99 Anomaly Test Samples\n",
      "Train-Val-Test Ratio: 0.666-0.141-0.193\n"
     ]
    }
   ],
   "source": [
    "# define train params\n",
    "target_size = (256, 256, 3)\n",
    "print(os.path.join(args.datadir, \"no_damage.txt\"))\n",
    "\n",
    "train_file_names, validation_file_names, normal_test_file_names, anomaly_train_file_names, anomaly_test_file_names = load_data(os.path.join(args.datadir, \"no_damage.txt\"),\n",
    "                                                                                                                                                       os.path.join(args.datadir, \"damage_encoded.txt\"))\n",
    "# train_generator, validation_generator, test_generator = preproc_data(\n",
    "#     input_shape, batch_size, args.datadir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_img_generator(file_paths, batch_size, target_size, preproc=True):\n",
    "    while True:\n",
    "        inds = (np.random.randint(0, len(file_paths), batch_size))\n",
    "        imgs = np.array([])\n",
    "        for i in inds:\n",
    "            fp = file_paths[i]\n",
    "            # print(fp)\n",
    "            if gfile.Exists(fp):\n",
    "                img = filepath_to_image(fp)\n",
    "                img = cv2.resize(img, target_size[:-1])\n",
    "                if preproc:\n",
    "                    np_img = preproc_img(img, flip_top_bottom=False, hsv=None)\n",
    "                else:\n",
    "                    np_img = np.array(img) / 255\n",
    "\n",
    "                imgs = np.concatenate(\n",
    "                    (imgs, np.array([np_img]))) if imgs.size > 0 else np.array([np_img])\n",
    "            yield imgs, imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_generator(file_paths, n, target_size):\n",
    "    # returns generator of n images\n",
    "    if len(file_paths) < n:\n",
    "        # prevent an IndexError\n",
    "        n = len(file_paths)\n",
    "    for i in range(n):\n",
    "        fp = file_paths[i]\n",
    "        if gfile.Exists(fp):\n",
    "            img = filepath_to_image(fp)\n",
    "            # img = img.resize(target_size[:-1], Image.BICUBIC)\n",
    "            img = cv2.resize(img, target_size[:-1])\n",
    "            yield np.array(img) / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filepath_to_image(fp):\n",
    "    with gfile.GFile(fp, mode=\"rb\") as f:\n",
    "        # img = Image.open(f)\n",
    "        img = cv2.imdecode(np.asarray(bytearray(f.read())), 1)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # convert2rgb\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rand(a=0, b=1):\n",
    "    \"\"\":returns random value between a and b\"\"\"\n",
    "    return np.random.rand()*(b-a) + a\n",
    "\n",
    "\n",
    "def preproc_img(image, flip_top_bottom=True, flip_left_right=True, hsv=(.1, 1.5, 1.5)):\n",
    "    \"\"\"https://github.com/qqwweee/keras-yolo3/blob/master/yolo3/utils.py\"\"\"\n",
    "    # flip image or not\n",
    "    if flip_left_right and rand() < .5:\n",
    "        # image = image.transpose(Image.FLIP_LEFT_RIGHT)\n",
    "        image = cv2.flip(image, flipCode=0)\n",
    "    if flip_top_bottom and rand() < .5:\n",
    "        # image = image.transpose(Image.FLIP_TOP_BOTTOM)\n",
    "        image = cv2.flip(image, flipCode=1)\n",
    "\n",
    "    # image brighDtness enhancer\n",
    "    # if brightness:\n",
    "        # brightness = ImageEnhance.Brightness(image)\n",
    "\n",
    "        # image = brightness.enhance(rand(.5, 1.5))\n",
    "        # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    # brightness and contrast\n",
    "\n",
    "    image = np.array(image)\n",
    "    image = np.clip((rand(.5, 1.5) * image + rand(-50, 100)), 0, 255)\n",
    "    image = image.astype(np.uint8)\n",
    "\n",
    "    # # increase or decrease contrast\n",
    "    # if contrast:\n",
    "    #     contrast = ImageEnhance.Contrast(image)\n",
    "    #     image = contrast.enhance(rand(.5, 1.5))\n",
    "\n",
    "    image = image / 255.\n",
    "\n",
    "    if hsv != None:\n",
    "        hue, sat, val = hsv\n",
    "        # distort image\n",
    "        hue = rand(-hue, hue)\n",
    "        sat = rand(1, sat) if rand() < .5 else 1/rand(1, sat)\n",
    "        val = rand(1, val) if rand() < .5 else 1 / rand(1, val)\n",
    "\n",
    "        x = rgb_to_hsv(image)\n",
    "        x[..., 0] += hue\n",
    "        x[..., 0][x[..., 0] > 1] -= 1\n",
    "        x[..., 0][x[..., 0] < 0] += 1\n",
    "        x[..., 1] *= sat\n",
    "        x[..., 2] *= val\n",
    "        x[x > 1] = 1\n",
    "        x[x < 0] = 0\n",
    "        image = hsv_to_rgb(x)\n",
    "    return image  # numpy array, 0 to 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_ae(autoencoder):\n",
    "    encoder_layer = autoencoder.get_layer(\"encoder\")\n",
    "    # this model maps an input to its encoded representation; Big image to small rep\n",
    "    encoder = Model(\n",
    "        inputs=autoencoder.get_input_at(0), outputs=encoder_layer.output)\n",
    "\n",
    "    # create a placeholder for an encoded (ENCODING_DIM-dimensional) input\n",
    "    encoded_input = Input(shape=encoder_layer.output_shape[1:])\n",
    "\n",
    "    # getting the middle of the autoencoder\n",
    "    start = (len(autoencoder.layers))//2\n",
    "    decoder = autoencoder.layers[-start](encoded_input)\n",
    "    # stacking the decoder layers\n",
    "    for i in range(start-1, 0, -1):\n",
    "        decoder = autoencoder.layers[-i](decoder)\n",
    "\n",
    "    # create the decoder model; \"<\": encoded(small) representation to big image\n",
    "    decoder = Model(encoded_input, decoder)\n",
    "    return encoder, decoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_conv_ae(filters, input_shape=(256, 256, 3)):\n",
    "    n = 10\n",
    "    print(filters)\n",
    "    if len(filters) != n:\n",
    "        raise ValueError(\"%d Filters must be given. Sorry.\" % n)\n",
    "    # this is our input placeholder\n",
    "    input_img = Input(shape=input_shape)\n",
    "    # layer between input and middle layer\n",
    "    i = 0\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2), padding=\"same\"\n",
    "    )(input_img)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (3, 3), strides=(1, 1),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (3, 3), strides=(1, 1),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (3, 3), strides=(1, 1),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    encode = Conv2D(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(encode)\n",
    "    encode = LeakyReLU(0.2)(encode)\n",
    "    i += 1\n",
    "    # \"encoded\" is the encoded representation of the input, middle layer of the aue\n",
    "    encoded = Conv2D(\n",
    "        filters[i], (4, 4), strides=(1, 1),  name=\"encoder\"\n",
    "    )(encode)\n",
    "\n",
    "    i -= 1\n",
    "    # layer between middle and output layer\n",
    "    decode = Conv2DTranspose(filters[i], (4, 4), strides=(1, 1), activation=\"relu\")(\n",
    "        encoded\n",
    "    )\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (3, 3), strides=(1, 1),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (3, 3), strides=(1, 1),  padding=\"same\"\n",
    "    )(decode)\n",
    "    i -= 1\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    i -= 1\n",
    "    decode = Conv2DTranspose(\n",
    "        filters[i], (4, 4), strides=(2, 2),  padding=\"same\"\n",
    "    )(decode)\n",
    "    decode = LeakyReLU(0.2)(decode)\n",
    "    \n",
    "    decoded = Conv2D(\n",
    "        input_shape[-1], (3, 3), activation=\"sigmoid\", padding=\"same\"\n",
    "    )(decode)\n",
    "\n",
    "    # this model maps an input to its reconstruction\n",
    "    autoencoder_single = Model(inputs=input_img, outputs=decoded)\n",
    "    n_gpus = len(get_available_gpus())\n",
    "    if n_gpus > 1:\n",
    "        autoencoder = multi_gpu_model(autoencoder_single, n_gpus)\n",
    "        print(\"Autoencoder is trained on %d GPUs\" % n_gpus)\n",
    "    else:\n",
    "        autoencoder = autoencoder_single\n",
    "        print(\"Autoencoder is trained on a single GPU\")\n",
    "    \n",
    "    encoder, decoder = split_ae(autoencoder_single)\n",
    "\n",
    "    # build (aka \"compile\") the model\n",
    "    autoencoder.compile(optimizer=\"adam\", loss=\"mse\")\n",
    "    autoencoder_single.compile(optimizer=\"adam\", loss=\"mse\")\n",
    "    return autoencoder, autoencoder_single, encoder, decoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_ae(train_file_names, validation_file_names,  args,  target_size=(256, 256, 3)):\n",
    "    print(\"Writing logs to %s\" % args.logdir)\n",
    "    train_generator = train_img_generator(\n",
    "        train_file_names, args.batch_size, target_size)\n",
    "    validation_generator = train_img_generator(\n",
    "        validation_file_names, args.batch_size, target_size, preproc=False)\n",
    "\n",
    "    print(\"Training Autoencoder for %s feature extraction...\" % USE_CASE)\n",
    "    d = args.ldim\n",
    "    batch_size = args.batch_size\n",
    "    filters = args.filters + [d]  # [16, 16, 16, 32, 64, 64, 32, 32, d]\n",
    "\n",
    "    ae, ae_single, encoder, decoder = build_conv_ae(\n",
    "        input_shape=target_size, filters=filters)\n",
    "    # define callbacks for logging and optimized training and ckpt saving\n",
    "\n",
    "    earlyStopping = EarlyStopping(\n",
    "        monitor=\"val_loss\", patience=20, verbose=1, mode=\"min\", min_delta=(1/10**5)\n",
    "    )\n",
    "    # saves model which was trained on a single cpu since saving the other model threw some kind of error\n",
    "    checkpoint_format = os.path.join(\n",
    "        args.ckptdir, 'ep{epoch:04d}-loss{loss:.6f}-val_loss{val_loss:.6f}.h5')\n",
    "\n",
    "    mcp_save = ModelCheckpointGC(\n",
    "        checkpoint_format, ae_single, save_best_only=True, verbose=1, monitor=\"val_loss\", mode=\"min\"\n",
    "    )\n",
    "    reduce_lr_loss = ReduceLROnPlateau(\n",
    "        monitor=\"val_loss\", factor=0.2, patience=7, verbose=1, mode=\"min\"\n",
    "    )\n",
    "\n",
    "    tb = TensorBoard(\n",
    "        log_dir=args.logdir, histogram_freq=0, write_graph=True, write_images=True\n",
    "    )\n",
    "\n",
    "    f = io.StringIO()\n",
    "    with redirect_stdout(f):\n",
    "        ae_single.summary()\n",
    "    summary = f.getvalue()\n",
    "    print(summary)\n",
    "\n",
    "    with gfile.GFile(os.path.join(args.logdir, \"train-specs.txt\"), mode=\"w\") as f:\n",
    "        f.write(\"\\n\".join((\"Filters: {}\".format(args.filters),\n",
    "                           \"Batch Size: %d\" % batch_size,\n",
    "                           \"Latent Space Dim: %d\" % d,\n",
    "                           \"Auto Encoder Network %s\" % summary)))\n",
    "\n",
    "    train_length = len(train_file_names)\n",
    "    val_length = len(validation_file_names)\n",
    "\n",
    "    ae.fit_generator(\n",
    "        generator=train_generator,  # needs to produce data infinitely\n",
    "        epochs=args.epochs,\n",
    "        # every element once on average\n",
    "        steps_per_epoch=train_length//batch_size,\n",
    "        validation_data=validation_generator,\n",
    "        validation_steps=val_length//batch_size,\n",
    "        callbacks=[mcp_save, earlyStopping, reduce_lr_loss, tb]\n",
    "    )\n",
    "\n",
    "    return ae, ae_single,encoder, decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(ckpt_path):\n",
    "    print(\"Loading Autoencoder for %s feature extraction from directory %s...\" % (\n",
    "        USE_CASE, ckpt_path))\n",
    "\n",
    "    url = urlparse(ckpt_path)  # [:-1] cuts the \\n from the lines\n",
    "    bucket = \"%s://%s\" % (url.scheme, url.netloc)\n",
    "\n",
    "    # [1:] removes leading / to avoid empty string in dir_parts\n",
    "    dir_parts = os.path.normpath(url.path[1:]).split(os.sep)\n",
    "\n",
    "    fp = os.path.join(\"tmp\", dir_parts[-1])\n",
    "    if not os.path.isdir(\"tmp\"):\n",
    "        os.mkdir(\"tmp\")\n",
    "\n",
    "    if not os.path.isfile(fp):\n",
    "        subprocess.call([\"gsutil\", \"-m\", \"cp\", ckpt_path, \"tmp\"])\n",
    "    ae = load_model(fp)\n",
    "\n",
    "    # this needs to be compiled since the untrained, single-GPU model is saved instead of the multi-GPU model\n",
    "    ae.compile(optimizer=\"adam\", loss=\"mse\")\n",
    "    encoder, decoder = split_ae(ae)\n",
    "    return ae, encoder, decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing logs to gs://e4u-anomaly-detection/logs/DEBUG_AD\n",
      "Training Autoencoder for spm feature extraction...\n",
      "[32, 32, 32, 64, 64, 128, 256, 128, 64, 256]\n",
      "Autoencoder is trained on 2 GPUs\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_13 (InputLayer)        (None, 256, 256, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_41 (Conv2D)           (None, 128, 128, 32)      1568      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_73 (LeakyReLU)   (None, 128, 128, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_42 (Conv2D)           (None, 64, 64, 32)        16416     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_74 (LeakyReLU)   (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_43 (Conv2D)           (None, 64, 64, 32)        9248      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_75 (LeakyReLU)   (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_44 (Conv2D)           (None, 32, 32, 64)        32832     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_76 (LeakyReLU)   (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_45 (Conv2D)           (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_77 (LeakyReLU)   (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_46 (Conv2D)           (None, 16, 16, 128)       131200    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_78 (LeakyReLU)   (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_47 (Conv2D)           (None, 8, 8, 256)         524544    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_79 (LeakyReLU)   (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_48 (Conv2D)           (None, 8, 8, 128)         295040    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_80 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_49 (Conv2D)           (None, 4, 4, 64)          131136    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_81 (LeakyReLU)   (None, 4, 4, 64)          0         \n",
      "_________________________________________________________________\n",
      "encoder (Conv2D)             (None, 1, 1, 256)         262400    \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_37 (Conv2DT (None, 4, 4, 64)          262208    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_82 (LeakyReLU)   (None, 4, 4, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_38 (Conv2DT (None, 8, 8, 128)         131200    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_83 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_39 (Conv2DT (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_84 (LeakyReLU)   (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_40 (Conv2DT (None, 16, 16, 128)       524416    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_85 (LeakyReLU)   (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_41 (Conv2DT (None, 32, 32, 64)        131136    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_86 (LeakyReLU)   (None, 32, 32, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_42 (Conv2DT (None, 64, 64, 64)        65600     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_87 (LeakyReLU)   (None, 64, 64, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_43 (Conv2DT (None, 64, 64, 32)        18464     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_88 (LeakyReLU)   (None, 64, 64, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_44 (Conv2DT (None, 128, 128, 32)      16416     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_89 (LeakyReLU)   (None, 128, 128, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_45 (Conv2DT (None, 256, 256, 32)      16416     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_90 (LeakyReLU)   (None, 256, 256, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_50 (Conv2D)           (None, 256, 256, 3)       867       \n",
      "=================================================================\n",
      "Total params: 2,903,203\n",
      "Trainable params: 2,903,203\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "Epoch 1/50\n",
      "45/45 [==============================] - 13s 296ms/step - loss: 0.0761 - val_loss: 0.0208\n",
      "Epoch 00000: val_loss improved from inf to 0.02077, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0000-loss0.064818-val_loss0.020767.h5\n",
      "Epoch 2/50\n",
      "45/45 [==============================] - 26s 568ms/step - loss: 0.0286 - val_loss: 0.0154\n",
      "Epoch 00001: val_loss improved from 0.02077 to 0.01544, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0001-loss0.027981-val_loss0.015439.h5\n",
      "Epoch 3/50\n",
      "45/45 [==============================] - 33s 741ms/step - loss: 0.0204 - val_loss: 0.0138\n",
      "Epoch 00002: val_loss improved from 0.01544 to 0.01379, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0002-loss0.020666-val_loss0.013791.h5\n",
      "Epoch 4/50\n",
      "45/45 [==============================] - 12s 274ms/step - loss: 0.0152 - val_loss: 0.0140\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 5/50\n",
      "45/45 [==============================] - 30s 656ms/step - loss: 0.0131 - val_loss: 0.0139\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 6/50\n",
      "45/45 [==============================] - 32s 700ms/step - loss: 0.0230 - val_loss: 0.0532\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 7/50\n",
      "45/45 [==============================] - 16s 361ms/step - loss: 0.0169 - val_loss: 0.0221\n",
      "Epoch 00006: val_loss did not improve\n",
      "Epoch 8/50\n",
      "45/45 [==============================] - 33s 742ms/step - loss: 0.0142 - val_loss: 0.0189\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 9/50\n",
      "45/45 [==============================] - 28s 619ms/step - loss: 0.0106 - val_loss: 0.0198\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 10/50\n",
      "45/45 [==============================] - 20s 450ms/step - loss: 0.0085 - val_loss: 0.0174\n",
      "Epoch 00009: val_loss did not improve\n",
      "\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.\n",
      "Epoch 11/50\n",
      "45/45 [==============================] - 37s 824ms/step - loss: 0.0092 - val_loss: 0.0163\n",
      "Epoch 00010: val_loss did not improve\n",
      "Epoch 12/50\n",
      "45/45 [==============================] - 25s 547ms/step - loss: 0.0110 - val_loss: 0.0163\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 13/50\n",
      "45/45 [==============================] - 24s 540ms/step - loss: 0.0118 - val_loss: 0.0158\n",
      "Epoch 00012: val_loss did not improve\n",
      "Epoch 14/50\n",
      "45/45 [==============================] - 41s 904ms/step - loss: 0.0143 - val_loss: 0.0146\n",
      "Epoch 00013: val_loss did not improve\n",
      "Epoch 15/50\n",
      "45/45 [==============================] - 17s 382ms/step - loss: 0.0121 - val_loss: 0.0149\n",
      "Epoch 00014: val_loss did not improve\n",
      "Epoch 16/50\n",
      "45/45 [==============================] - 21s 468ms/step - loss: 0.0087 - val_loss: 0.0086\n",
      "Epoch 00015: val_loss improved from 0.01379 to 0.00863, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0015-loss0.008693-val_loss0.008629.h5\n",
      "Epoch 17/50\n",
      "45/45 [==============================] - 37s 825ms/step - loss: 0.0090 - val_loss: 0.0083\n",
      "Epoch 00016: val_loss improved from 0.00863 to 0.00831, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0016-loss0.008996-val_loss0.008306.h5\n",
      "Epoch 18/50\n",
      "45/45 [==============================] - 12s 260ms/step - loss: 0.0116 - val_loss: 0.0088\n",
      "Epoch 00017: val_loss did not improve\n",
      "Epoch 19/50\n",
      "45/45 [==============================] - 26s 578ms/step - loss: 0.0096 - val_loss: 0.0077\n",
      "Epoch 00018: val_loss improved from 0.00831 to 0.00766, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0018-loss0.009598-val_loss0.007658.h5\n",
      "Epoch 20/50\n",
      "45/45 [==============================] - 39s 857ms/step - loss: 0.0101 - val_loss: 0.0092\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 21/50\n",
      "45/45 [==============================] - 13s 281ms/step - loss: 0.0098 - val_loss: 0.0073\n",
      "Epoch 00020: val_loss improved from 0.00766 to 0.00728, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0020-loss0.009429-val_loss0.007283.h5\n",
      "Epoch 22/50\n",
      "45/45 [==============================] - 30s 664ms/step - loss: 0.0102 - val_loss: 0.0086\n",
      "Epoch 00021: val_loss did not improve\n",
      "Epoch 23/50\n",
      "45/45 [==============================] - 36s 790ms/step - loss: 0.0093 - val_loss: 0.0092\n",
      "Epoch 00022: val_loss did not improve\n",
      "Epoch 24/50\n",
      "45/45 [==============================] - 16s 365ms/step - loss: 0.0080 - val_loss: 0.0088\n",
      "Epoch 00023: val_loss did not improve\n",
      "Epoch 25/50\n",
      "45/45 [==============================] - 33s 740ms/step - loss: 0.0087 - val_loss: 0.0087\n",
      "Epoch 00024: val_loss did not improve\n",
      "Epoch 26/50\n",
      "45/45 [==============================] - 32s 700ms/step - loss: 0.0089 - val_loss: 0.0100\n",
      "Epoch 00025: val_loss did not improve\n",
      "Epoch 27/50\n",
      "45/45 [==============================] - 20s 449ms/step - loss: 0.0081 - val_loss: 0.0096\n",
      "Epoch 00026: val_loss did not improve\n",
      "Epoch 28/50\n",
      "45/45 [==============================] - 37s 824ms/step - loss: 0.0078 - val_loss: 0.0077\n",
      "Epoch 00027: val_loss did not improve\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05.\n",
      "Epoch 29/50\n",
      "45/45 [==============================] - 25s 546ms/step - loss: 0.0085 - val_loss: 0.0071\n",
      "Epoch 00028: val_loss improved from 0.00728 to 0.00707, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0028-loss0.008037-val_loss0.007071.h5\n",
      "Epoch 30/50\n",
      "45/45 [==============================] - 18s 409ms/step - loss: 0.0097 - val_loss: 0.0084\n",
      "Epoch 00029: val_loss did not improve\n",
      "Epoch 31/50\n",
      "45/45 [==============================] - 35s 770ms/step - loss: 0.0097 - val_loss: 0.0088\n",
      "Epoch 00030: val_loss did not improve\n",
      "Epoch 32/50\n",
      "45/45 [==============================] - 18s 410ms/step - loss: 0.0115 - val_loss: 0.0076\n",
      "Epoch 00031: val_loss did not improve\n",
      "Epoch 33/50\n",
      "45/45 [==============================] - 22s 497ms/step - loss: 0.0111 - val_loss: 0.0085\n",
      "Epoch 00032: val_loss did not improve\n",
      "Epoch 34/50\n",
      "45/45 [==============================] - 39s 860ms/step - loss: 0.0094 - val_loss: 0.0077\n",
      "Epoch 00033: val_loss did not improve\n",
      "Epoch 35/50\n",
      "45/45 [==============================] - 15s 336ms/step - loss: 0.0087 - val_loss: 0.0077\n",
      "Epoch 00034: val_loss did not improve\n",
      "Epoch 36/50\n",
      "45/45 [==============================] - 26s 582ms/step - loss: 0.0083 - val_loss: 0.0077\n",
      "Epoch 00035: val_loss did not improve\n",
      "\n",
      "Epoch 00036: ReduceLROnPlateau reducing learning rate to 8.000000525498762e-06.\n",
      "Epoch 37/50\n",
      "45/45 [==============================] - 42s 925ms/step - loss: 0.0090 - val_loss: 0.0077\n",
      "Epoch 00036: val_loss did not improve\n",
      "Epoch 38/50\n",
      "45/45 [==============================] - 13s 281ms/step - loss: 0.0079 - val_loss: 0.0079\n",
      "Epoch 00037: val_loss did not improve\n",
      "Epoch 39/50\n",
      "45/45 [==============================] - 30s 664ms/step - loss: 0.0082 - val_loss: 0.0078\n",
      "Epoch 00038: val_loss did not improve\n",
      "Epoch 40/50\n",
      "45/45 [==============================] - 38s 846ms/step - loss: 0.0100 - val_loss: 0.0080\n",
      "Epoch 00039: val_loss did not improve\n",
      "Epoch 41/50\n",
      "45/45 [==============================] - 16s 366ms/step - loss: 0.0114 - val_loss: 0.0076\n",
      "Epoch 00040: val_loss did not improve\n",
      "Epoch 42/50\n",
      "45/45 [==============================] - 34s 745ms/step - loss: 0.0106 - val_loss: 0.0075\n",
      "Epoch 00041: val_loss did not improve\n",
      "Epoch 43/50\n",
      "45/45 [==============================] - 33s 727ms/step - loss: 0.0098 - val_loss: 0.0075\n",
      "Epoch 00042: val_loss did not improve\n",
      "\n",
      "Epoch 00043: ReduceLROnPlateau reducing learning rate to 1.6000001778593287e-06.\n",
      "Epoch 44/50\n",
      "45/45 [==============================] - 14s 300ms/step - loss: 0.0113 - val_loss: 0.0072\n",
      "Epoch 00043: val_loss did not improve\n",
      "Epoch 45/50\n",
      "45/45 [==============================] - 29s 646ms/step - loss: 0.0104 - val_loss: 0.0064\n",
      "Epoch 00044: val_loss improved from 0.00707 to 0.00640, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0044-loss0.010332-val_loss0.006396.h5\n",
      "Epoch 46/50\n",
      "45/45 [==============================] - 20s 434ms/step - loss: 0.0111 - val_loss: 0.0062\n",
      "Epoch 00045: val_loss improved from 0.00640 to 0.00620, saving model to gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0045-loss0.010420-val_loss0.006200.h5\n",
      "Epoch 47/50\n",
      "45/45 [==============================] - 18s 405ms/step - loss: 0.0114 - val_loss: 0.0064\n",
      "Epoch 00046: val_loss did not improve\n",
      "Epoch 48/50\n",
      "45/45 [==============================] - 35s 781ms/step - loss: 0.0101 - val_loss: 0.0066\n",
      "Epoch 00047: val_loss did not improve\n",
      "Epoch 49/50\n",
      "45/45 [==============================] - 22s 489ms/step - loss: 0.0075 - val_loss: 0.0065\n",
      "Epoch 00048: val_loss did not improve\n",
      "Epoch 50/50\n",
      "45/45 [==============================] - 22s 500ms/step - loss: 0.0069 - val_loss: 0.0067\n",
      "Epoch 00049: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "ae_multi,ae, _, _ = train_ae(train_file_names, validation_file_names,  args,  target_size=target_size)\n",
    "# ae, _, _ = load(\n",
    "#        \"gs://e4u-anomaly-detection/ckpts/AD_spm_20190813_144154/ep0057-loss0.009716-val_loss0.015439.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_from_generator(model, gen, batch_size):\n",
    "    predictions = []\n",
    "    for _ in gen:\n",
    "        batch = []\n",
    "        for _ in range(batch_size):\n",
    "            try:\n",
    "                batch.append(next(gen))\n",
    "            except StopIteration:\n",
    "                pass\n",
    "        pred = model.predict(np.array(batch))\n",
    "        # pred[0], because prediction happens on an array, hence result has also length one\n",
    "        predictions.extend(pred)\n",
    "        print(len(predictions), \"predictions made\")\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(ae, loss_boundary, oc_svm,  X_normal_test, X_anomaly_test, img_dir, target_size):\n",
    "    # load the data for evaluation purposes\n",
    "    print(\"Evaluating feature extractor...\")\n",
    "    # eval_train = model.evaluate_generator(\n",
    "    #     train_img_generator(X_normal_train, 8), steps=len(X_normal_train))\n",
    "    batch_size = 64\n",
    "\n",
    "    eval_test = ae.evaluate_generator(\n",
    "        train_img_generator(X_normal_test, batch_size, target_size, preproc=False), steps=len(X_normal_test)//batch_size)\n",
    "    # print(\"Feature extractor train loss: %f\" % eval_train)\n",
    "\n",
    "    print(\"Feature extractor  test loss: %f\" % eval_test)\n",
    "\n",
    "    decoded_samples_normal = predict_from_generator(\n",
    "        ae, img_generator(X_normal_test, 8, target_size),batch_size)\n",
    "\n",
    "    plot_samples(\n",
    "        img_generator(X_normal_test, 8, target_size),\n",
    "        decoded_samples_normal,\n",
    "        plot_mvtec,\n",
    "        os.path.join(img_dir, \"rec-normals.png\"),\n",
    "    )\n",
    "\n",
    "    decoded_samples_anomaly = predict_from_generator(\n",
    "        ae, img_generator(X_anomaly_test, batch_size, target_size),batch_size)\n",
    "\n",
    "    plot_samples(\n",
    "        img_generator(X_anomaly_test, batch_size, target_size),\n",
    "        decoded_samples_anomaly,\n",
    "        plot_mvtec,\n",
    "        os.path.join(img_dir, \"rec-anomalies.png\"),\n",
    "    )\n",
    "\n",
    "    metrics = {}\n",
    "\n",
    "    print(\"Evaluating the loss based approach\")\n",
    "    metrics[\"loss\"] = eval_loss(model=ae, X_normal=X_normal_test,\n",
    "                                X_anomaly=X_anomaly_test, loss_boundary=loss_boundary, img_dir=img_dir, target_size=target_size)\n",
    "    print(\"Evaluating the OC SVM\")\n",
    "\n",
    "    encoder, _ = split_ae(ae)\n",
    "    metrics[\"svm\"] = eval_svm(encoder, oc_svm,\n",
    "                              X_normal_test, X_anomaly_test, img_dir, target_size, batch_size)\n",
    "    print(metrics)\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(TP, TN, FP, FN):\n",
    "    try:\n",
    "        precision = TP / (TP + FP)\n",
    "    except ZeroDivisionError:\n",
    "        precision = 0\n",
    "    try:\n",
    "        recall = TP / (TP + FN)\n",
    "    except ZeroDivisionError:\n",
    "        recall = 0\n",
    "\n",
    "    accuracy = (TN + TP) / (TP + TN + FN + FP)\n",
    "    TPR = recall\n",
    "    try:\n",
    "        TNR = TN / (TN + FP)\n",
    "    except ZeroDivisionError:\n",
    "        TNR = 0\n",
    "    \n",
    "    FNR = 1 - TPR\n",
    "    FPR = 1 - TNR\n",
    "    \n",
    "    try:\n",
    "        F_1 = 2 * (precision * recall) / (precision + recall)\n",
    "    except ZeroDivisionError:\n",
    "        F_1 = 0\n",
    "    try:\n",
    "        F_2 = 5 * (precision * recall) / (4*precision + recall)\n",
    "    except ZeroDivisionError:\n",
    "        F_2 = 0\n",
    "\n",
    "    try:\n",
    "        # between -1 and 1\n",
    "        MCC = (TP*TN - FP*FN)/(np.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)))\n",
    "    except ZeroDivisionError:\n",
    "        MCC = 0\n",
    "    current_metrics = {\"precision\": precision,\n",
    "                       \"accuracy\": accuracy,\n",
    "                       \"recall\": recall,\n",
    "                       \"F1_score\": F_1,\n",
    "                       \"F2_score\": F_2,\n",
    "                       \"TPR\": TPR,\n",
    "                       \"TNR\": TNR,\n",
    "                       \"FNR\": FNR,\n",
    "                       \"FPR\": FPR,\n",
    "                       \"MCC\": MCC\n",
    "                       }\n",
    "    return current_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ROC_curve(metrics, file_path):\n",
    "    # TODO: fix plot saving\n",
    "    fig = plt.figure()\n",
    "    plt.plot([m[\"FPR\"] for m in metrics],\n",
    "             [m[\"TPR\"] for m in metrics])\n",
    "    plt.grid()\n",
    "    plt.xlabel(\"FALSE POSITIVE RATE\")\n",
    "    plt.ylabel(\"TRUE POSITIVE RATE\")\n",
    "    savefig(fig, file_path)\n",
    "\n",
    "    plt.clf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_per_img(img, rec_img):\n",
    "    # mean squared error\n",
    "    return np.sum(np.power(rec_img - img, 2))  # / (np.prod(img.shape) - 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_losses(model, X_normal, X_anomaly, target_size):\n",
    "    batch_size = 64\n",
    "    print(\"Computing losses for normal samples.\")\n",
    "    samples_normal = img_generator(\n",
    "        X_normal, len(X_normal), target_size\n",
    "    )\n",
    "    print(\"Reconstructing normal images.\")\n",
    "    decoded_samples_normal = predict_from_generator(model, samples_normal,batch_size=batch_size)\n",
    "    print(\"Computing normal losses per image\")\n",
    "    normal_losses = np.array([loss_per_img(i, ri)for i, ri in zip(\n",
    "        img_generator(X_normal, len(X_normal), target_size), decoded_samples_normal)])\n",
    "    \n",
    "    print(\"Computing losses for anomalous samples.\")\n",
    "    print(\"Reconstructing anomalous images.\")\n",
    "    decoded_samples_anomaly = predict_from_generator(model, img_generator(\n",
    "        X_anomaly, len(X_anomaly), target_size\n",
    "    ),batch_size=batch_size)\n",
    "    \n",
    "    print(\"Computing anomaly losses per image\")\n",
    "    anomaly_losses = np.array([loss_per_img(i, ri)for i, ri in zip(\n",
    "        img_generator(X_anomaly, len(X_anomaly), target_size), decoded_samples_anomaly)])\n",
    "    return normal_losses, anomaly_losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_loss(model, X_normal, X_anomaly, loss_boundary, img_dir, target_size):\n",
    "    \"\"\"anomaly detection based on the loss\"\"\"\n",
    "\n",
    "    # compute losses on the test set\n",
    "    normal_losses, anomaly_losses = get_losses(model,\n",
    "                                               X_normal, X_anomaly, target_size)\n",
    "    bins = 10\n",
    "    # loss distribution over the normal dataset\n",
    "    fig = plt.figure()\n",
    "    label = \"Distribution of normal loss values\"\n",
    "    plot_hist(normal_losses, relative=True,\n",
    "              color=\"g\", bins=bins, label=label)\n",
    "\n",
    "    label = \"Distribution of normal loss values\"\n",
    "    # loss distribution over the anomaly dataset\n",
    "    plot_hist(anomaly_losses, relative=True,\n",
    "              color=\"r\", bins=bins, label=label)\n",
    "\n",
    "    savefig(fig, os.path.join(img_dir, \"loss-dist.png\"))\n",
    "    plt.clf()\n",
    "\n",
    "    # ground truth: negatives = anomaly\n",
    "    TP = anomaly_losses[anomaly_losses >= loss_boundary]\n",
    "    FN = anomaly_losses[anomaly_losses < loss_boundary]\n",
    "\n",
    "    # ground truth: positives = normality\n",
    "    TN = normal_losses[normal_losses < loss_boundary]\n",
    "    FP = normal_losses[normal_losses >= loss_boundary]\n",
    "\n",
    "    # ROC_curve(metrics, os.path.join(img_dir, \"loss-ROC.png\"))\n",
    "    return get_metrics(len(TP), len(TN), len(FP), len(FN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_svm(encoder, oc_svm,   X_normal, X_anomaly, img_dir, target_size, batch_size):\n",
    "\n",
    "    print(\"Encoding normal test images to latent space...\")\n",
    "    encoded_normal_imgs_test = encoder.predict_generator(\n",
    "        train_img_generator(X_normal, batch_size, target_size, preproc=False), steps=len(X_normal)/batch_size)\n",
    "    print(\"Encoding anomaly test images to latent space...\")\n",
    "    encoded_anomaly_imgs_test = encoder.predict_generator(\n",
    "        train_img_generator(X_anomaly, batch_size, target_size, preproc=False), steps=len(X_anomaly)/batch_size)\n",
    "\n",
    "    # reshape to suitable shape for OCC\n",
    "    encoded_normal_imgs_test = encoded_normal_imgs_test.reshape(\n",
    "        -1, np.prod(encoded_normal_imgs_test.shape[1:]))\n",
    "    encoded_anomaly_imgs_test = encoded_anomaly_imgs_test.reshape(\n",
    "        - 1, np.prod(encoded_anomaly_imgs_test.shape[1:]))\n",
    "\n",
    "    score_normal = oc_svm.predict(encoded_normal_imgs_test)\n",
    "    score_anomaly = oc_svm.predict(encoded_anomaly_imgs_test)\n",
    "\n",
    "    bins = 10\n",
    "    # loss distribution over the normal dataset\n",
    "    fig = plt.figure()\n",
    "    label = \"Distribution of the normal svm score values\"\n",
    "    plot_hist(oc_svm.decision_function(encoded_normal_imgs_test), relative=True,\n",
    "              color=\"g\", bins=bins, label=label)\n",
    "\n",
    "    label = \"Distribution of normal svm score values\"\n",
    "    # loss distribution over the anomaly dataset\n",
    "    plot_hist(oc_svm.decision_function(encoded_anomaly_imgs_test), relative=True,\n",
    "              color=\"r\", bins=bins, label=label)\n",
    "\n",
    "    savefig(fig, os.path.join(img_dir, \"svm-score-dist.png\"))\n",
    "    # clear figure\n",
    "    plt.clf()\n",
    "\n",
    "    # Normalities are negatives (no finding)\n",
    "\n",
    "    # ground truth: anomaly\n",
    "    TP = len(score_anomaly[score_anomaly == -1])\n",
    "    FN = len(score_anomaly[score_anomaly == 1])\n",
    "\n",
    "    # ground truth: Normality\n",
    "    TN = len(score_normal[score_normal == 1])\n",
    "    FP = len(score_normal[score_normal == -1])\n",
    "\n",
    "    # ROC_curve(metrics, os.path.join(img_dir, \"svm-ROC.png\"))\n",
    "    return get_metrics(TP, TN, FP, FN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_loss(model, normal_paths, anomaly_paths, target_size):\n",
    "    print(\"Training the loss.\")\n",
    "    normal_losses, anomaly_losses = get_losses(\n",
    "        model, normal_paths, anomaly_paths, target_size)\n",
    "    # generate new samples of normals and anomalies to determine the best boundary on the train set\n",
    "    # samples_normal = np.array(\n",
    "    #    list(img_generator(normal_paths, len(normal_paths), target_size)))  \n",
    "    # samples_normal_generator = img_generator(normal_paths, len(normal_paths), target_size)\n",
    "    # samples_anomaly = np.array(\n",
    "    #   list(img_generator(anomaly_paths, len(anomaly_paths), target_size)))\n",
    "    \n",
    "    best_m = -1\n",
    "    best_boundary = 0\n",
    "    print(\"Determining best loss boundary for train set.\")\n",
    "    decisive_metric = \"MCC\"\n",
    "    for loss_boundary in normal_losses:\n",
    "        # ground truth: positives = normality\n",
    "        TP = normal_losses[normal_losses < loss_boundary]\n",
    "        FN = normal_losses[normal_losses >= loss_boundary]\n",
    "\n",
    "        # ground truth: negatives = anomaly\n",
    "        TN = anomaly_losses[anomaly_losses >= loss_boundary]\n",
    "        FP = anomaly_losses[anomaly_losses < loss_boundary]\n",
    "        \n",
    "        current_metrics = get_metrics(\n",
    "            len(TP), len(TN), len(FP), len(FN))\n",
    "\n",
    "        # less or equal since we want the biggest TP_rate (i)\n",
    "        if current_metrics[decisive_metric] >= best_m:\n",
    "            best_m = current_metrics[decisive_metric]\n",
    "            best_boundary = loss_boundary\n",
    "            print(\"New best metric:\",best_m,\"Boundary Loss = \", best_boundary)\n",
    "\n",
    "    return best_boundary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs://training-data-images/out/P04033/1-37029 1-37028/back_1m50.jpg',\n",
       " 'gs://training-data-images/out/P04033/1-37029 1-37028/front_1m50.jpg',\n",
       " 'gs://training-data-images/out/P04033/1-37031 1-37032/front_19m90.jpg',\n",
       " 'gs://training-data-images/out/P04033/1-37031 1-37032/front_22m85.jpg',\n",
       " 'gs://training-data-images/out/P04033/1-37031 1-37032/front_31m55.jpg']"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anomaly_train_file_names[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the loss.\n",
      "Computing losses for normal samples.\n",
      "Reconstructing normal images.\n",
      "64 predictions made\n",
      "128 predictions made\n",
      "192 predictions made\n",
      "256 predictions made\n",
      "320 predictions made\n",
      "384 predictions made\n",
      "448 predictions made\n",
      "512 predictions made\n",
      "576 predictions made\n",
      "640 predictions made\n",
      "704 predictions made\n",
      "768 predictions made\n",
      "832 predictions made\n",
      "896 predictions made\n",
      "960 predictions made\n",
      "1024 predictions made\n",
      "1088 predictions made\n",
      "1152 predictions made\n",
      "1216 predictions made\n",
      "1280 predictions made\n",
      "1344 predictions made\n",
      "1408 predictions made\n",
      "1472 predictions made\n",
      "1536 predictions made\n",
      "1600 predictions made\n",
      "1664 predictions made\n",
      "1728 predictions made\n",
      "1792 predictions made\n",
      "1856 predictions made\n",
      "1920 predictions made\n",
      "1984 predictions made\n",
      "2048 predictions made\n",
      "2112 predictions made\n",
      "2176 predictions made\n",
      "2240 predictions made\n",
      "2304 predictions made\n",
      "2368 predictions made\n",
      "2432 predictions made\n",
      "2496 predictions made\n",
      "2560 predictions made\n",
      "2624 predictions made\n",
      "2688 predictions made\n",
      "2752 predictions made\n",
      "2816 predictions made\n",
      "2880 predictions made\n",
      "2944 predictions made\n",
      "3008 predictions made\n",
      "3072 predictions made\n",
      "3136 predictions made\n",
      "3200 predictions made\n",
      "3264 predictions made\n",
      "3328 predictions made\n",
      "3392 predictions made\n",
      "3456 predictions made\n",
      "3520 predictions made\n",
      "3584 predictions made\n",
      "3648 predictions made\n",
      "3712 predictions made\n",
      "3776 predictions made\n",
      "3840 predictions made\n",
      "3904 predictions made\n",
      "3968 predictions made\n",
      "4032 predictions made\n",
      "4096 predictions made\n",
      "4160 predictions made\n",
      "4224 predictions made\n",
      "4288 predictions made\n",
      "4352 predictions made\n",
      "4416 predictions made\n",
      "4480 predictions made\n",
      "4544 predictions made\n",
      "4608 predictions made\n",
      "4672 predictions made\n",
      "4736 predictions made\n",
      "4800 predictions made\n",
      "4864 predictions made\n",
      "4928 predictions made\n",
      "4992 predictions made\n",
      "5056 predictions made\n",
      "5120 predictions made\n",
      "5184 predictions made\n",
      "5248 predictions made\n",
      "5312 predictions made\n",
      "5376 predictions made\n",
      "5440 predictions made\n",
      "5504 predictions made\n",
      "5568 predictions made\n",
      "5632 predictions made\n",
      "5696 predictions made\n",
      "5720 predictions made\n",
      "Computing normal losses per image\n",
      "Computing losses for anomalous samples.\n",
      "Reconstructing anomalous images.\n",
      "64 predictions made\n",
      "128 predictions made\n",
      "192 predictions made\n",
      "256 predictions made\n",
      "320 predictions made\n",
      "384 predictions made\n",
      "445 predictions made\n",
      "Computing anomaly losses per image\n",
      "Determining best loss boundary for train set.\n",
      "New best metric: 0.030282445698499787 Boundary Loss =  3675.5832529489326\n",
      "New best metric: 0.03362713651455768 Boundary Loss =  3600.4328797527146\n",
      "New best metric: 0.035762065390348347 Boundary Loss =  3837.0156401487448\n",
      "New best metric: 0.037262393398916176 Boundary Loss =  3401.4187608401244\n",
      "New best metric: 0.04344665008552865 Boundary Loss =  3340.4338968172774\n",
      "New best metric: 0.044807289849531536 Boundary Loss =  2929.2651769051777\n",
      "New best metric: 0.04518713530833397 Boundary Loss =  2997.8963654964964\n",
      "New best metric: 0.05862412782119923 Boundary Loss =  1824.5913421991424\n",
      "New best metric: 0.06377874025259749 Boundary Loss =  1798.8555177283058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:32: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1798.8555177283058"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_boundary = train_loss(\n",
    "    ae, train_file_names, anomaly_train_file_names, target_size)\n",
    "loss_boundary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_svm(model, normal_file_names,  target_size, batch_size):\n",
    "    print(\"Training the OC SVM\")\n",
    "    print(\"Encoding train images to latent space...\")\n",
    "    # encode normal instances to latent space\n",
    "    encoded_normal_imgs_train = model.predict_generator(\n",
    "        train_img_generator(normal_file_names, batch_size, target_size, preproc=False), steps=len(normal_file_names) / batch_size)  # used later for One Class Classification\n",
    "\n",
    "    encoded_normal_imgs_train = encoded_normal_imgs_train.reshape(\n",
    "        - 1, np.prod(encoded_normal_imgs_train.shape[1:]))\n",
    "\n",
    "    # # encode anomalies to latent space\n",
    "    # encoded_anomaly_imgs_train = model.predict_generator(\n",
    "    #     train_img_generator(anomaly_file_names, batch_size, target_size, preproc=False), steps=len(anomaly_file_names)/batch_size)  # used later for One Class Classification\n",
    "    # encoded_anomaly_imgs_train = encoded_anomaly_imgs_train.reshape(\n",
    "    #     - 1, np.prod(encoded_anomaly_imgs_train.shape[1:]))\n",
    "\n",
    "    print(\"Fitting the OC SVM\")\n",
    "    clf = svm.OneClassSVM()\n",
    "    # TODO: evaluate different nu values\n",
    "    clf.fit(encoded_normal_imgs_train)\n",
    "\n",
    "    # # computes signed distance to the hyperplane\n",
    "    # score_normal = clf.predict(encoded_normal_imgs_train)\n",
    "    # score_anomaly = clf.predict(encoded_anomaly_imgs_train)\n",
    "\n",
    "    # # ground truth: anomaly\n",
    "    # TP = len(score_anomaly[score_anomaly == -1])\n",
    "    # FN = len(score_anomaly[score_anomaly == 1])\n",
    "\n",
    "    # # ground truth: Normality\n",
    "    # TN = len(score_normal[score_normal == 1])\n",
    "    # FP = len(score_normal[score_normal == -1])\n",
    "\n",
    "    # current_metrics = get_metrics(TP, TN, FP, FN)\n",
    "    # print(\"SVM train results:\", current_metrics)\n",
    "\n",
    "    return clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the OC SVM\n",
      "Encoding train images to latent space...\n",
      "Fitting the OC SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "encoder = split_ae(ae)[0]\n",
    "oc_svm = train_svm(encoder, train_file_names,\n",
    "                       target_size, batch_size=args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating feature extractor...\n",
      "Feature extractor  test loss: 0.012908\n",
      "7 predictions made\n",
      "63 predictions made\n",
      "Evaluating the loss based approach\n",
      "Computing losses for normal samples.\n",
      "Reconstructing normal images.\n",
      "64 predictions made\n",
      "128 predictions made\n",
      "192 predictions made\n",
      "256 predictions made\n",
      "320 predictions made\n",
      "384 predictions made\n",
      "448 predictions made\n",
      "512 predictions made\n",
      "576 predictions made\n",
      "640 predictions made\n",
      "704 predictions made\n",
      "768 predictions made\n",
      "832 predictions made\n",
      "896 predictions made\n",
      "960 predictions made\n",
      "1024 predictions made\n",
      "1088 predictions made\n",
      "1152 predictions made\n",
      "1216 predictions made\n",
      "1280 predictions made\n",
      "1344 predictions made\n",
      "1408 predictions made\n",
      "1472 predictions made\n",
      "1536 predictions made\n",
      "1600 predictions made\n",
      "1659 predictions made\n",
      "Computing normal losses per image\n",
      "Computing losses for anomalous samples.\n",
      "Reconstructing anomalous images.\n",
      "64 predictions made\n",
      "97 predictions made\n",
      "Computing anomaly losses per image\n",
      "Evaluating the OC SVM\n",
      "Encoding normal test images to latent space...\n",
      "Encoding anomaly test images to latent space...\n",
      "{'svm': {'TPR': 1.0, 'TNR': 0.06084656084656084, 'FNR': 0.0, 'FPR': 0.9391534391534392, 'F1_score': 0.016620498614958446, 'precision': 0.008379888268156424, 'F2_score': 0.040540540540540536, 'MCC': 0.022580686025800042, 'recall': 1.0, 'accuracy': 0.06824146981627296}, 'loss': {'TPR': 0.9381443298969072, 'TNR': 0.059674502712477394, 'FNR': 0.061855670103092786, 'FPR': 0.9403254972875226, 'F1_score': 0.10411899313501144, 'precision': 0.05511811023622047, 'F2_score': 0.22314860225600783, 'MCC': -0.0021015023024030413, 'recall': 0.9381443298969072, 'accuracy': 0.1082004555808656}}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1152x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1152x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# returns metrics dictionary\n",
    "metrics = evaluate(ae, loss_boundary, oc_svm, normal_test_file_names,\n",
    "                       anomaly_test_file_names, args.imgdir, target_size)\n",
    "with gfile.GFile(os.path.join(args.evaldir, \"metrics.json\"), \"w\") as f:\n",
    "    json.dump(metrics, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0000-loss0.056312-val_loss0.018705.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0000-loss0.061939-val_loss0.050991.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0000-loss0.064818-val_loss0.020767.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0000-loss0.094021-val_loss0.092254.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0001-loss0.024045-val_loss0.012379.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0001-loss0.027981-val_loss0.015439.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0001-loss0.028694-val_loss0.038799.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0001-loss0.039759-val_loss0.049159.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0002-loss0.020666-val_loss0.013791.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0002-loss0.020791-val_loss0.024569.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0003-loss0.012724-val_loss0.011841.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0003-loss0.014298-val_loss0.020205.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0004-loss0.013616-val_loss0.016352.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0004-loss0.016842-val_loss0.035506.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0003-loss0.016572-val_loss0.043635.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0006-loss0.015640-val_loss0.014185.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0007-loss0.020951-val_loss0.023561.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0007-loss0.016610-val_loss0.013966.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0006-loss0.027168-val_loss0.033500.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0008-loss0.017868-val_loss0.019672.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0009-loss0.011791-val_loss0.017655.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0010-loss0.011533-val_loss0.011468.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0010-loss0.014622-val_loss0.016632.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0012-loss0.010738-val_loss0.011252.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0013-loss0.008540-val_loss0.010050.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0013-loss0.009565-val_loss0.009829.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0015-loss0.008693-val_loss0.008629.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0015-loss0.012033-val_loss0.007905.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0016-loss0.008996-val_loss0.008306.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0018-loss0.009598-val_loss0.007658.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0020-loss0.009429-val_loss0.007283.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0028-loss0.008037-val_loss0.007071.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0044-loss0.010332-val_loss0.006396.h5...\n",
      "Copying gs://e4u-anomaly-detection/ckpts/DEBUG_AD/ep0045-loss0.010420-val_loss0.006200.h5...\n",
      "Copying gs://e4u-anomaly-detection/eval/DEBUG_AD/metrics.json...                \n",
      "Copying gs://e4u-anomaly-detection/imgs/DEBUG_AD/loss-dist.png...               \n",
      "Copying gs://e4u-anomaly-detection/imgs/DEBUG_AD/rec-anomalies.png...           \n",
      "Copying gs://e4u-anomaly-detection/imgs/DEBUG_AD/svm-score-dist.png...          \n",
      "Copying gs://e4u-anomaly-detection/logs/DEBUG_AD/events.out.tfevents.1566204469.e4u-anomaly-detection...\n",
      "Copying gs://e4u-anomaly-detection/logs/DEBUG_AD/events.out.tfevents.1566212541.e4u-anomaly-detection...\n",
      "Copying gs://e4u-anomaly-detection/imgs/DEBUG_AD/rec-normals.png...             \n",
      "Copying gs://e4u-anomaly-detection/logs/DEBUG_AD/events.out.tfevents.1566215890.e4u-anomaly-detection...\n",
      "Copying gs://e4u-anomaly-detection/logs/DEBUG_AD/train-specs.txt...             \n",
      "/ [43/44 files][693.3 MiB/693.3 MiB]  99% Done                                  \r"
     ]
    }
   ],
   "source": [
    "!gsutil -m cp -r gs://e4u-anomaly-detection/*/DEBUG_AD ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}