week3.5/fix_my_nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from lasagne.layers import *\n",
    "from lasagne.nonlinearities import *\n",
    "from lasagne import init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nn = InputLayer([None,3,100,100])\n",
    "\n",
    "nn = Conv2DLayer(nn,num_filters=512, filter_size=(3,3),\n",
    "                 W = init.Constant(0))\n",
    "\n",
    "nn = Conv2DLayer(nn,num_filters=128,filter_size=(3,3),\n",
    "                 W = init.Constant(0))\n",
    "\n",
    "nn = Conv2DLayer(nn,num_filters=32,filter_size=(3,3),\n",
    "                 W = init.Constant(0))\n",
    "\n",
    "nn = Pool2DLayer(nn,pool_size=(6,6),mode='max')\n",
    "\n",
    "nn = Conv2DLayer(nn,num_filters=8,filter_size=(10,10),\n",
    "                W = init.Normal(std=0.01))\n",
    "\n",
    "nn = Conv2DLayer(nn,num_filters=8,filter_size=(10,10),\n",
    "                W = init.Normal(std=0.01))\n",
    "\n",
    "nn = Pool2DLayer(nn,pool_size=(3,3),mode='max')\n",
    "\n",
    "nn = DenseLayer(nn,512,nonlinearity=softmax)\n",
    "\n",
    "nn = DropoutLayer(nn,p=0.5)\n",
    "\n",
    "nn = DenseLayer(nn,512,nonlinearity=softmax)\n",
    "\n",
    "nn = DenseLayer(nn,10,nonlinearity=sigmoid)\n",
    "\n",
    "nn = DropoutLayer(nn,p=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "# Book of grudges\n",
    "* zero init for weights will cause symmetry effect\n",
    "* Too many filters for first 3x3 convolution - will lead to enormous matrix while there's just not enough relevant combinations of 3x3 images (overkill).\n",
    "* Usually the further you go, the more filters you need.\n",
    "* large filters (10x10 is generally a bad pactice, and you definitely need more than 10 of them\n",
    "* the second of 10x10 convolution gets 8x6x6 image as input, so it's technically unable to perform such convolution.\n",
    "* Softmax nonlinearity effectively makes only 1 or a few neurons from the entire layer to \"fire\", rendering 512-neuron layer almost useless. Softmax at the output layer is okay though\n",
    "* Dropout after probability prediciton is just lame. A few random classes get probability of 0, so your probabilities no longer sum to 1 and crossentropy goes -inf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}