ipynb/ex3_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "A notebook to process experimental results of ex2_prob_params.py. p(reject) as problem parameters are varied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "#%config InlineBackend.figure_format = 'svg'\n",
    "#%config InlineBackend.figure_format = 'pdf'\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import kgof.data as data\n",
    "import kgof.glo as glo\n",
    "import kgof.goftest as gof\n",
    "import kgof.kernel as kernel\n",
    "import kgof.plot as plot\n",
    "import kgof.util as util\n",
    "\n",
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import kgof.plot\n",
    "kgof.plot.set_default_matplotlib_options()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# np.random.seed(0)\n",
    "# x = np.linspace(-5., 5., 50)\n",
    "# y = 3 * np.exp(-0.5 * (x - 1.3)**2 / 0.8**2)\n",
    "# y += np.random.normal(0., 0.2, x.shape)\n",
    "# f = plt.figure(0)\n",
    "# plt.plot(x,y, 'b-')\n",
    "# plt.xlabel(r'$\\alpha$')\n",
    "# plt.ylabel('Test power')\n",
    "# f.savefig('test.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_plot_vs_Js(fname, show_legend=True, xscale='log', yscale='linear'):\n",
    "    \"\"\"\n",
    "    J = number of test locations.\n",
    "    \"\"\"\n",
    "    func_xvalues = lambda agg_results: agg_results['Js']\n",
    "    ex = 3\n",
    "    def func_title(agg_results):\n",
    "        repeats, _, n_methods = agg_results['job_results'].shape\n",
    "        alpha = agg_results['alpha']\n",
    "        test_size = (1.0 - agg_results['tr_proportion'])*agg_results['sample_size']\n",
    "        title = '%s. %d trials. test size: %d. $\\\\alpha$ = %.2g.'%\\\n",
    "            ( agg_results['prob_label'], repeats, test_size, alpha)\n",
    "        return title\n",
    "    #plt.figure(figsize=(10,5))\n",
    "    results = plot.plot_prob_reject(\n",
    "        ex, fname, func_xvalues, '', func_title=func_title)\n",
    "    \n",
    "    plt.title('')\n",
    "    \n",
    "    if xscale is not None:\n",
    "        plt.xscale(xscale)\n",
    "    if yscale is not None:\n",
    "        plt.yscale(yscale)\n",
    "    \n",
    "    plt.xlabel('$J$')\n",
    "    plt.gca().legend(loc='best').set_visible(show_legend)\n",
    "    if show_legend:\n",
    "        plt.legend(bbox_to_anchor=(1.70, 1.05))\n",
    "        \n",
    "    plt.grid(False)\n",
    "        \n",
    "    return results\n",
    "\n",
    "\n",
    "def load_runtime_vs_Js(fname, xlabel='$J$ parameter', \n",
    "                      show_legend=True, xscale='linear', yscale='linear'):\n",
    "    func_xvalues = lambda agg_results: agg_results['Js']\n",
    "    ex = 3\n",
    "    def func_title(agg_results):\n",
    "        repeats, _, n_methods = agg_results['job_results'].shape\n",
    "        alpha = agg_results['alpha']\n",
    "        title = '%s. %d trials. $\\\\alpha$ = %.2g.'%\\\n",
    "            ( agg_results['prob_label'], repeats, alpha)\n",
    "        return title\n",
    "    \n",
    "    #plt.figure(figsize=(10,6))\n",
    "    \n",
    "    results = plot.plot_runtime(ex, fname,  \n",
    "                                func_xvalues, xlabel=xlabel, func_title=func_title)\n",
    "    \n",
    "    plt.title('')\n",
    "    plt.gca().legend(loc='best').set_visible(show_legend)\n",
    "    if show_legend:\n",
    "        plt.legend(bbox_to_anchor=(1.70, 1.05))\n",
    "    \n",
    "    plt.grid(False)\n",
    "    if xscale is not None:\n",
    "        plt.xscale(xscale)\n",
    "    if yscale is not None:\n",
    "        plt.yscale(yscale)\n",
    "        \n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GMD\n",
    "# gmd_fname = 'ex3-gmd1-me2_n500_rs100_Jmi2_Jma32_a0.050_trp0.50.p'\n",
    "# gmd_results = load_plot_vs_Js(gmd_fname, show_legend=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# p: normal, q: Gaussian mixture\n",
    "# g_vs_gmm_fname = 'ex3-g_vs_gmm_d5-me2_n500_rs50_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "# g_vs_gmm_fname = 'ex3-g_vs_gmm_d5-me2_n500_rs100_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "# g_vs_gmm_fname = 'ex3-g_vs_gmm_d2-me2_n500_rs50_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "# g_vs_gmm_fname = 'ex3-g_vs_gmm_d1-me2_n500_rs50_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "g_vs_gmm_fname = 'ex3-g_vs_gmm_d1-me2_n500_rs200_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "# g_vs_gmm_fname = 'ex3-g_vs_gmm_d1-me2_n800_rs50_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "g_vs_gmm_results = load_plot_vs_Js(g_vs_gmm_fname, show_legend=False)\n",
    "plt.xticks([1, 10, 1e2, 1e3])\n",
    "plt.savefig(g_vs_gmm_fname.replace('.p', '.pdf', 1), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gaussian mixture\n",
    "# gmm_fname = 'ex3-gmm_d1-me2_n500_rs100_Jmi2_Jma32_a0.050_trp0.50.p'\n",
    "# gmm_results = load_plot_vs_Js(gmm_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Same Gaussian\n",
    "sg5_fname = \"ex3-sg5-me2_n500_rs100_Jmi2_Jma384_a0.050_trp0.50.p\"\n",
    "sg5_results = load_plot_vs_Js(sg5_fname, show_legend=False)\n",
    "plt.ylim([0, 0.05])\n",
    "plt.xticks([1, 10, 1e2, 1e3])\n",
    "plt.savefig(sg5_fname.replace('.p', '.pdf', 1), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p(x)=\\mathcal{N}(0, I) \\\\\n",
    "q(x)=\\mathcal{N}(0, I)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Gaussian variance difference. \n",
    "gvd5_fname = 'ex3-gvd5-me2_n500_rs100_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "gvd5_results = load_plot_vs_Js(gvd5_fname, show_legend=True)\n",
    "plt.legend(bbox_to_anchor=(1.8, 1.05))\n",
    "plt.xticks([1, 10, 1e2, 1e3])\n",
    "plt.savefig(gvd5_fname.replace('.p', '.pdf', 1), bbox_inches='tight')\n",
    "# plt.legend(ncol=2)\n",
    "#plt.ylim([0.03, 0.1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p(x)=\\mathcal{N}(0, I) \\\\\n",
    "q(x)=\\mathcal{N}(0, \\mathrm{diag}(2,1,1,\\ldots))$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gauss-Bernoulli RBM. H1 case\n",
    "# rbm_h1_fname = 'ex3-gbrbm_dx5_dh3_v5em3-me2_n500_rs100_Jmi2_Jma384_a0.050_trp0.50.p'\n",
    "# rbm_h1_results = load_plot_vs_Js(rbm_h1_fname, show_legend=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}