notebooks/eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_latest_checkpoint_from_dir(dir):\n",
    "    largest_epoch = 0\n",
    "    largest_epoch_fn = None\n",
    "    for fn in os.listdir(dir):\n",
    "        if fn.endswith(\".ckpt\") and fn != \"last.ckpt\":\n",
    "            base = os.path.basename(fn)[:-5]\n",
    "            parts = base.split(\"-\")\n",
    "            epoch = int(parts[0].split(\"=\")[1])\n",
    "            if epoch > largest_epoch:\n",
    "                largest_epoch = epoch\n",
    "                largest_epoch_fn = fn\n",
    "    return os.path.join(dir, largest_epoch_fn), largest_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_checkpoints_from_dir(dir):\n",
    "    for fn in os.listdir(dir):\n",
    "        if fn.endswith(\".ckpt\") and fn != \"last.ckpt\":\n",
    "            base = os.path.basename(fn)[:-5]\n",
    "            parts = base.split(\"-\")\n",
    "            epoch = int(parts[0].split(\"=\")[1])\n",
    "            yield os.path.join(dir, fn), epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deeplabv3+_resnet50_tmax_200\n",
      "deeplabv3+_resnext50_32x4d_tmax_200\n",
      "default_deeplabv3+_resnet18\n",
      "default_deeplabv3+_resnet50\n",
      "default_unet_resnet18\n",
      "default_unet_resnet50\n",
      "fcn_128_0.001\n",
      "fcn_64_0.001\n",
      "fcn_64_0.003\n",
      "fcn_64_0.01\n",
      "unet_resnet50_tmax_300\n",
      "unet_resnet50_tmax_50\n"
     ]
    }
   ],
   "source": [
    "!ls -1 logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_dirs = [\n",
    "    \"logs/default_unet/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_unet_resnet18/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_unet_resnet50_lr_0.01/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_deeplabv3+_resnet18/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_deeplabv3+_resnet50/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_customfcn/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_customfcn_lr_0.01/lightning_logs/version_0/checkpoints/\",\n",
    "    \"logs/default_fcn/lightning_logs/version_0/checkpoints/\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    \"deeplabv3+_resnet50_tmax_200\",\n",
    "    \"deeplabv3+_resnext50_32x4d_tmax_200\",\n",
    "    \"default_deeplabv3+_resnet18\",\n",
    "    \"default_deeplabv3+_resnet50\",\n",
    "    \"default_unet_resnet18\",\n",
    "    \"default_unet_resnet50\",\n",
    "    \"fcn_128_0.001\",\n",
    "    \"fcn_64_0.001\",\n",
    "    \"fcn_64_0.003\",\n",
    "    \"fcn_64_0.01\",\n",
    "    \"unet_resnet50_tmax_300\",\n",
    "    \"unet_resnet50_tmax_50\",\n",
    "]\n",
    "experiment_dirs = [\n",
    "    f\"logs/{experiment}/lightning_logs/version_0/checkpoints/\"\n",
    "    for experiment in experiments\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deeplabv3+_resnet50_tmax_200 95\n",
      "0.994617707221794,0.9919556599491307,0.8359778017134237,0.7210724220887,0.6171346069026353\n",
      "deeplabv3+_resnext50_32x4d_tmax_200 87\n",
      "0.9943122916834607,0.9922722546059191,0.8455158811499861,0.716889685957076,0.6202604683537325\n",
      "default_deeplabv3+_resnet18 54\n",
      "0.994411884482251,0.9914393907741712,0.8261434669167365,0.7188247341063101,0.5893881285147204\n",
      "default_deeplabv3+_resnet50 143\n",
      "0.9945757587407563,0.9925026101156188,0.8482257687169799,0.7198290076130116,0.6392782073547248\n",
      "default_unet_resnet18 140\n",
      "0.9942197019677801,0.9920841816137448,0.8358377836753835,0.7105452308298627,0.6350601982026685\n",
      "default_unet_resnet50 148\n",
      "0.9945358579448385,0.9922245356787779,0.8404084117092925,0.7178818252436012,0.634580714521998\n",
      "fcn_128_0.001 147\n",
      "0.9938146870252207,0.9830802074436191,0.6408220963754082,0.7151669172518368,0.23441118094608004\n",
      "fcn_64_0.001 147\n",
      "0.9938164718139928,0.9811900649465972,0.5916397784868177,0.7096343767213815,0.18105169395743742\n",
      "fcn_64_0.003 145\n",
      "0.9937525206260713,0.9813631351962718,0.5956076320347924,0.707776551080016,0.1883206183151395\n",
      "fcn_64_0.01 149\n",
      "0.9940609482431614,0.9792479571892166,0.5401608691103356,0.7093212256580733,0.12838481365089865\n",
      "unet_resnet50_tmax_300 249\n",
      "0.9942796647935231,0.9924314574662499,0.8433702472693209,0.7113368372156088,0.6500089590914103\n",
      "unet_resnet50_tmax_50 199\n",
      "0.9954259034379153,0.9893330122468299,0.7742216528202273,0.7385831112559527,0.5212557889513728\n"
     ]
    }
   ],
   "source": [
    "for experiment_dir in experiment_dirs:\n",
    "    model_fn, epoch = get_latest_checkpoint_from_dir(experiment_dir)\n",
    "    print(experiment_dir.split(\"/\")[1], epoch)\n",
    "    command = [\n",
    "        \"python\", \"eval.py\",\n",
    "        \"--model_fn\", model_fn,\n",
    "        \"--eval_set\", \"test\",\n",
    "        \"--gpu\", \"0\",\n",
    "        \"--quiet\",\n",
    "    ]\n",
    "    subprocess.call(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unet_resnet50_tmax_50 49\n",
      "0.9949269658027166,0.9905324866975009,0.8036428427488459,0.7283915732786309,0.5607646929099129\n",
      "unet_resnet50_tmax_50 99\n",
      "0.994109765056417,0.9913746964444519,0.8258513930907793,0.7138187994517393,0.5823862539971331\n",
      "unet_resnet50_tmax_50 149\n",
      "0.9947860731766911,0.9917966020598897,0.8306468947399299,0.7230657520382783,0.6177267683868122\n",
      "unet_resnet50_tmax_50 199\n",
      "0.9940948301659143,0.9922834854305236,0.8412662597794575,0.7088574217085203,0.6392945749255706\n",
      "unet_resnet50_tmax_50 249\n",
      "0.9942796647935231,0.9924314574662499,0.8433702472693209,0.7113368372156088,0.6500089590914103\n"
     ]
    }
   ],
   "source": [
    "for model_fn, epoch in get_all_checkpoints_from_dir(\"logs/unet_resnet50_tmax_300/lightning_logs/version_0/checkpoints/\"):\n",
    "    print(experiment_dir.split(\"/\")[1], epoch)\n",
    "    command = [\n",
    "        \"python\", \"eval.py\",\n",
    "        \"--model_fn\", model_fn,\n",
    "        \"--eval_set\", \"test\",\n",
    "        \"--gpu\", \"0\",\n",
    "        \"--quiet\",\n",
    "    ]\n",
    "    subprocess.call(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "landcover",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}