examples/2.eval-cvrplib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate on CVRPLib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-A.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-B.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-E.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-F.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-M.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-P.tgz\n",
      "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-X.tgz\n"
     ]
    }
   ],
   "source": [
    "# If you did not download the VRPLIB, you may run the following\n",
    "from routefinder.data.download_vrplib import download_vrplib\n",
    "\n",
    "download_vrplib() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/lightning/pytorch/utilities/parsing.py:208: Attribute 'env' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['env'])`.\n",
      "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/lightning/pytorch/utilities/parsing.py:208: Attribute 'policy' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['policy'])`.\n",
      "Provided file name ['data/cvrp/val/100.npz', 'data/ovrp/val/100.npz', 'data/ovrpb/val/100.npz', 'data/ovrpbl/val/100.npz', 'data/ovrpbltw/val/100.npz', 'data/ovrpbtw/val/100.npz', 'data/ovrpl/val/100.npz', 'data/ovrpltw/val/100.npz', 'data/ovrptw/val/100.npz', 'data/vrpb/val/100.npz', 'data/vrpl/val/100.npz', 'data/vrpbltw/val/100.npz', 'data/vrpbtw/val/100.npz', 'data/vrpbl/val/100.npz', 'data/vrpltw/val/100.npz', 'data/vrptw/val/100.npz', 'data/cvrp/val/50.npz', 'data/vrptw/val/50.npz'] not found. Make sure to provide a file in the right path first or unset val_file to generate data automatically instead\n",
      "Provided file name ['data/cvrp/test/100.npz', 'data/ovrp/test/100.npz', 'data/ovrpb/test/100.npz', 'data/ovrpbl/test/100.npz', 'data/ovrpbltw/test/100.npz', 'data/ovrpbtw/test/100.npz', 'data/ovrpl/test/100.npz', 'data/ovrpltw/test/100.npz', 'data/ovrptw/test/100.npz', 'data/vrpb/test/100.npz', 'data/vrpl/test/100.npz', 'data/vrpbltw/test/100.npz', 'data/vrpbtw/test/100.npz', 'data/vrpbl/test/100.npz', 'data/vrpltw/test/100.npz', 'data/vrptw/test/100.npz', 'data/cvrp/test/50.npz', 'data/vrptw/test/50.npz'] not found. Make sure to provide a file in the right path first or unset test_file to generate data automatically instead\n",
      "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/rl4co/models/rl/reinforce/reinforce.py:204: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  state_dict = torch.load(checkpoint_path, map_location=map_location)[\"state_dict\"]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "from routefinder.models.model import RouteFinderBase, RouteFinderMoE\n",
    "from routefinder.models.baselines.mvmoe.model import MVMoE\n",
    "from routefinder.models.baselines.mtpomo.model import MTPOMO\n",
    "from routefinder.envs.mtvrp import MTVRPEnv, MTVRPGenerator\n",
    "\n",
    "\n",
    "# Choose your model\n",
    "\n",
    "PATH = \"../checkpoints/100/rf-transformer.ckpt\"\n",
    "model = RouteFinderBase.load_from_checkpoint(PATH, map_location=\"cpu\")\n",
    "\n",
    "# PATH = \"../checkpoints/100/mtpomo.ckpt\"\n",
    "# model = MTPOMO.load_from_checkpoint(PATH, map_location=\"cpu\")\n",
    "\n",
    "# PATH = \"../checkpoints/100/mvmoe.ckpt\"\n",
    "# model = MVMoE.load_from_checkpoint(PATH, map_location=\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = model.policy\n",
    "\n",
    "# Create env\n",
    "generator = MTVRPGenerator(num_loc=100, variant_preset=\"all\")\n",
    "env = MTVRPEnv(generator, check_solution=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average reward: -19.634\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "policy = policy.to(device).eval()\n",
    "\n",
    "td_test = env.reset(env.generator(32))\n",
    "\n",
    "# Test the model\n",
    "with torch.amp.autocast(\"cuda\"):\n",
    "    with torch.inference_mode():\n",
    "        out = policy(td_test.clone().to(device), env, phase=\"test\", decode_type=\"greedy\", return_actions=True)\n",
    "        actions = out['actions'].cpu().detach()\n",
    "        rewards = out['reward'].cpu().detach()\n",
    "\n",
    "print(f\"Average reward: {rewards.mean().item():.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VRPLib Evaluation\n",
    "\n",
    "Note: run \n",
    "\n",
    "```\n",
    "python routefinder/data/download_vrplib.py\n",
    "```\n",
    " to download the VRPLib instances (then move them under this dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found VRP file: A-n37-k6.vrp\n",
      "Found VRP file: A-n33-k5.vrp\n",
      "Found VRP file: A-n44-k6.vrp\n",
      "Found VRP file: A-n48-k7.vrp\n",
      "Found VRP file: A-n39-k5.vrp\n",
      "Found VRP file: A-n63-k10.vrp\n",
      "Found VRP file: A-n36-k5.vrp\n",
      "Found VRP file: A-n34-k5.vrp\n",
      "Found VRP file: A-n55-k9.vrp\n",
      "Found VRP file: A-n69-k9.vrp\n",
      "Found VRP file: A-n61-k9.vrp\n",
      "Found VRP file: A-n38-k5.vrp\n",
      "Found VRP file: A-n53-k7.vrp\n",
      "Found VRP file: A-n45-k6.vrp\n",
      "Found VRP file: A-n80-k10.vrp\n",
      "Found VRP file: A-n46-k7.vrp\n",
      "Found VRP file: A-n33-k6.vrp\n",
      "Found VRP file: A-n37-k5.vrp\n",
      "Found VRP file: A-n60-k9.vrp\n",
      "Found VRP file: A-n39-k6.vrp\n",
      "Found VRP file: A-n32-k5.vrp\n",
      "Found VRP file: A-n45-k7.vrp\n",
      "Found VRP file: A-n65-k9.vrp\n",
      "Found VRP file: A-n64-k9.vrp\n",
      "Found VRP file: A-n62-k8.vrp\n",
      "Found VRP file: A-n63-k9.vrp\n",
      "Found VRP file: A-n54-k7.vrp\n",
      "Found VRP file: B-n41-k6.vrp\n",
      "Found VRP file: B-n35-k5.vrp\n",
      "Found VRP file: B-n38-k6.vrp\n",
      "Found VRP file: B-n64-k9.vrp\n",
      "Found VRP file: B-n39-k5.vrp\n",
      "Found VRP file: B-n50-k8.vrp\n",
      "Found VRP file: B-n45-k6.vrp\n",
      "Found VRP file: B-n43-k6.vrp\n",
      "Found VRP file: B-n45-k5.vrp\n",
      "Found VRP file: B-n34-k5.vrp\n",
      "Found VRP file: B-n68-k9.vrp\n",
      "Found VRP file: B-n56-k7.vrp\n",
      "Found VRP file: B-n31-k5.vrp\n",
      "Found VRP file: B-n44-k7.vrp\n",
      "Found VRP file: B-n66-k9.vrp\n",
      "Found VRP file: B-n57-k7.vrp\n",
      "Found VRP file: B-n51-k7.vrp\n",
      "Found VRP file: B-n50-k7.vrp\n",
      "Found VRP file: B-n78-k10.vrp\n",
      "Found VRP file: B-n63-k10.vrp\n",
      "Found VRP file: B-n67-k10.vrp\n",
      "Found VRP file: B-n52-k7.vrp\n",
      "Found VRP file: B-n57-k9.vrp\n",
      "Found VRP file: E-n101-k8.vrp\n",
      "Found VRP file: E-n101-k14.vrp\n",
      "Found VRP file: E-n23-k3.vrp\n",
      "Found VRP file: E-n22-k4.vrp\n",
      "Found VRP file: E-n76-k7.vrp\n",
      "Found VRP file: E-n33-k4.vrp\n",
      "Found VRP file: E-n13-k4.vrp\n",
      "Found VRP file: E-n76-k14.vrp\n",
      "Found VRP file: E-n76-k8.vrp\n",
      "Found VRP file: E-n76-k10.vrp\n",
      "Found VRP file: E-n31-k7.vrp\n",
      "Found VRP file: E-n51-k5.vrp\n",
      "Found VRP file: E-n30-k3.vrp\n",
      "Found VRP file: F-n72-k4.vrp\n",
      "Found VRP file: F-n135-k7.vrp\n",
      "Found VRP file: F-n45-k4.vrp\n",
      "Found VRP file: M-n200-k17.vrp\n",
      "Found VRP file: M-n121-k7.vrp\n",
      "Found VRP file: M-n200-k16.vrp\n",
      "Found VRP file: M-n101-k10.vrp\n",
      "Found VRP file: M-n151-k12.vrp\n",
      "Found VRP file: P-n22-k2.vrp\n",
      "Found VRP file: P-n76-k4.vrp\n",
      "Found VRP file: P-n51-k10.vrp\n",
      "Found VRP file: P-n55-k7.vrp\n",
      "Found VRP file: P-n16-k8.vrp\n",
      "Found VRP file: P-n19-k2.vrp\n",
      "Found VRP file: P-n60-k15.vrp\n",
      "Found VRP file: P-n55-k8.vrp\n",
      "Found VRP file: P-n22-k8.vrp\n",
      "Found VRP file: P-n40-k5.vrp\n",
      "Found VRP file: P-n20-k2.vrp\n",
      "Found VRP file: P-n55-k15.vrp\n",
      "Found VRP file: P-n65-k10.vrp\n",
      "Found VRP file: P-n101-k4.vrp\n",
      "Found VRP file: P-n21-k2.vrp\n",
      "Found VRP file: P-n50-k8.vrp\n",
      "Found VRP file: P-n23-k8.vrp\n",
      "Found VRP file: P-n76-k5.vrp\n",
      "Found VRP file: P-n70-k10.vrp\n",
      "Found VRP file: P-n50-k7.vrp\n",
      "Found VRP file: P-n50-k10.vrp\n",
      "Found VRP file: P-n45-k5.vrp\n",
      "Found VRP file: P-n55-k10.vrp\n",
      "Found VRP file: P-n60-k10.vrp\n",
      "Found VRP file: X-n120-k6.vrp\n",
      "Found VRP file: X-n856-k95.vrp\n",
      "Found VRP file: X-n766-k71.vrp\n",
      "Found VRP file: X-n524-k153.vrp\n",
      "Found VRP file: X-n420-k130.vrp\n",
      "Found VRP file: X-n561-k42.vrp\n",
      "Found VRP file: X-n214-k11.vrp\n",
      "Found VRP file: X-n701-k44.vrp\n",
      "Found VRP file: X-n256-k16.vrp\n",
      "Found VRP file: X-n125-k30.vrp\n",
      "Found VRP file: X-n261-k13.vrp\n",
      "Found VRP file: X-n895-k37.vrp\n",
      "Found VRP file: X-n819-k171.vrp\n",
      "Found VRP file: X-n513-k21.vrp\n",
      "Found VRP file: X-n536-k96.vrp\n",
      "Found VRP file: X-n344-k43.vrp\n",
      "Found VRP file: X-n1001-k43.vrp\n",
      "Found VRP file: X-n439-k37.vrp\n",
      "Found VRP file: X-n586-k159.vrp\n",
      "Found VRP file: X-n401-k29.vrp\n",
      "Found VRP file: X-n641-k35.vrp\n",
      "Found VRP file: X-n783-k48.vrp\n",
      "Found VRP file: X-n480-k70.vrp\n",
      "Found VRP file: X-n251-k28.vrp\n",
      "Found VRP file: X-n237-k14.vrp\n",
      "Found VRP file: X-n613-k62.vrp\n",
      "Found VRP file: X-n228-k23.vrp\n",
      "Found VRP file: X-n101-k25.vrp\n",
      "Found VRP file: X-n336-k84.vrp\n",
      "Found VRP file: X-n233-k16.vrp\n",
      "Found VRP file: X-n266-k58.vrp\n",
      "Found VRP file: X-n294-k50.vrp\n",
      "Found VRP file: X-n209-k16.vrp\n",
      "Found VRP file: X-n393-k38.vrp\n",
      "Found VRP file: X-n469-k138.vrp\n",
      "Found VRP file: X-n351-k40.vrp\n",
      "Found VRP file: X-n139-k10.vrp\n",
      "Found VRP file: X-n289-k60.vrp\n",
      "Found VRP file: X-n270-k35.vrp\n",
      "Found VRP file: X-n685-k75.vrp\n",
      "Found VRP file: X-n599-k92.vrp\n",
      "Found VRP file: X-n749-k98.vrp\n",
      "Found VRP file: X-n134-k13.vrp\n",
      "Found VRP file: X-n548-k50.vrp\n",
      "Found VRP file: X-n223-k34.vrp\n",
      "Found VRP file: X-n502-k39.vrp\n",
      "Found VRP file: X-n106-k14.vrp\n",
      "Found VRP file: X-n359-k29.vrp\n",
      "Found VRP file: X-n162-k11.vrp\n",
      "Found VRP file: X-n115-k10.vrp\n",
      "Found VRP file: X-n655-k131.vrp\n",
      "Found VRP file: X-n936-k151.vrp\n",
      "Found VRP file: X-n573-k30.vrp\n",
      "Found VRP file: X-n176-k26.vrp\n",
      "Found VRP file: X-n186-k15.vrp\n",
      "Found VRP file: X-n376-k94.vrp\n",
      "Found VRP file: X-n153-k22.vrp\n",
      "Found VRP file: X-n303-k21.vrp\n",
      "Found VRP file: X-n167-k10.vrp\n",
      "Found VRP file: X-n322-k28.vrp\n",
      "Found VRP file: X-n110-k13.vrp\n",
      "Found VRP file: X-n449-k29.vrp\n",
      "Found VRP file: X-n317-k53.vrp\n",
      "Found VRP file: X-n200-k36.vrp\n",
      "Found VRP file: X-n143-k7.vrp\n",
      "Found VRP file: X-n733-k159.vrp\n",
      "Found VRP file: X-n429-k61.vrp\n",
      "Found VRP file: X-n157-k13.vrp\n",
      "Found VRP file: X-n181-k23.vrp\n",
      "Found VRP file: X-n491-k59.vrp\n",
      "Found VRP file: X-n204-k19.vrp\n",
      "Found VRP file: X-n331-k15.vrp\n",
      "Found VRP file: X-n801-k40.vrp\n",
      "Found VRP file: X-n172-k51.vrp\n",
      "Found VRP file: X-n837-k142.vrp\n",
      "Found VRP file: X-n195-k51.vrp\n",
      "Found VRP file: X-n670-k130.vrp\n",
      "Found VRP file: X-n327-k20.vrp\n",
      "Found VRP file: X-n627-k43.vrp\n",
      "Found VRP file: X-n298-k31.vrp\n",
      "Found VRP file: X-n284-k15.vrp\n",
      "Found VRP file: X-n190-k8.vrp\n",
      "Found VRP file: X-n148-k46.vrp\n",
      "Found VRP file: X-n876-k59.vrp\n",
      "Found VRP file: X-n313-k71.vrp\n",
      "Found VRP file: X-n280-k17.vrp\n",
      "Found VRP file: X-n275-k28.vrp\n",
      "Found VRP file: X-n247-k50.vrp\n",
      "Found VRP file: X-n219-k73.vrp\n",
      "Found VRP file: X-n129-k18.vrp\n",
      "Found VRP file: X-n242-k48.vrp\n",
      "Found VRP file: X-n411-k19.vrp\n",
      "Found VRP file: X-n367-k17.vrp\n",
      "Found VRP file: X-n459-k26.vrp\n",
      "Found VRP file: X-n979-k58.vrp\n",
      "Found VRP file: X-n716-k35.vrp\n",
      "Found VRP file: X-n957-k87.vrp\n",
      "Found VRP file: X-n384-k52.vrp\n",
      "Found VRP file: X-n916-k207.vrp\n",
      "Found VRP file: X-n308-k13.vrp\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Ensure you have downloaded vrplib under vrplib/\n",
    "# Initialize the instances dictionary\n",
    "instances = {}\n",
    "\n",
    "# Walk through the vrplib directory recursively\n",
    "for root, dirs, files in sorted(os.walk('./vrplib')):\n",
    "    for file in files:\n",
    "        if file.endswith('.vrp'):\n",
    "            # Initialize the dictionary for this instance\n",
    "            instance_name = file[:-4]  # Remove the '.vrp' extension\n",
    "            instances[instance_name] = {\"solution\": None}  # Create entry for instance\n",
    "            # Print the file for verification\n",
    "            instances[instance_name][\"data\"] = os.path.join(root, file)  # Save the VRP file path\n",
    "            instances[instance_name][\"solution\"] = os.path.join(root, file[:-4] + '.sol')  # Save the solution file path\n",
    "            # ensure the solution file exists\n",
    "            assert os.path.exists(instances[instance_name][\"solution\"]), f\"Solution file not found for {instance_name}\"\n",
    "            print(f\"Found VRP file: {file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rl4co.utils.ops import unbatchify, gather_by_index\n",
    "\n",
    "# Utils function\n",
    "def normalize_coord(coord:torch.Tensor) -> torch.Tensor: # if we scale x and y separately, aren't we losing the relative position of the points? i.e. we mess with the distances.\n",
    "    x, y = coord[:, 0], coord[:, 1]\n",
    "    x_min, x_max = x.min(), x.max()\n",
    "    y_min, y_max = y.min(), y.max()\n",
    "    \n",
    "    x_scaled = (x - x_min) / (x_max - x_min) \n",
    "    y_scaled = (y - y_min) / (y_max - y_min)\n",
    "    coord_scaled = torch.stack([x_scaled, y_scaled], dim=1)\n",
    "    return coord_scaled \n",
    "\n",
    "\n",
    "def evaluate(model, td, env,\n",
    "             num_augment=8,\n",
    "             num_starts=None,\n",
    "             ):\n",
    "    \n",
    "    with torch.inference_mode():\n",
    "        with torch.amp.autocast(\"cuda\"):\n",
    "            n_start = env.get_num_starts(td) if num_starts is None else num_starts\n",
    "\n",
    "            if num_augment > 1:\n",
    "                td = model.augment(td)\n",
    "\n",
    "            # Evaluate policy\n",
    "            out = model.policy(\n",
    "                td, env, phase=\"test\", num_starts=n_start, return_actions=True\n",
    "            )\n",
    "\n",
    "            # Unbatchify reward to [batch_size, num_augment, num_starts].\n",
    "            reward = unbatchify(out[\"reward\"], (num_augment, n_start))\n",
    "\n",
    "            if n_start > 1:\n",
    "                # max multi-start reward\n",
    "                max_reward, max_idxs = reward.max(dim=-1)\n",
    "                out.update({\"max_reward\": max_reward})\n",
    "\n",
    "                if out.get(\"actions\", None) is not None:\n",
    "                    # Reshape batch to [batch_size, num_augment, num_starts, ...]\n",
    "                    actions = unbatchify(out[\"actions\"], (num_augment, n_start))\n",
    "                    out.update(\n",
    "                        {\"best_multistart_actions\": gather_by_index(actions, max_idxs, dim=max_idxs.dim())}\n",
    "                    )\n",
    "                    out[\"actions\"] = actions\n",
    "\n",
    "            # Get augmentation score only during inference\n",
    "            if num_augment > 1:\n",
    "                # If multistart is enabled, we use the best multistart rewards\n",
    "                reward_ = max_reward if n_start > 1 else reward\n",
    "                max_aug_reward, max_idxs = reward_.max(dim=1)\n",
    "                out.update({\"max_aug_reward\": max_aug_reward})\n",
    "\n",
    "                if out.get(\"actions\", None) is not None:\n",
    "                    actions_ = (\n",
    "                        out[\"best_multistart_actions\"] if n_start > 1 else out[\"actions\"]\n",
    "                    )\n",
    "                    out.update({\"best_aug_actions\": gather_by_index(actions_, max_idxs)})\n",
    "                    \n",
    "            return out\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Problem: A-n37-k6        Cost: 970        Optimal Cost: 949       \t Gap: 2.213%\n",
      "Problem: A-n33-k5        Cost: 694        Optimal Cost: 661       \t Gap: 4.992%\n",
      "Problem: A-n44-k6        Cost: 988        Optimal Cost: 937       \t Gap: 5.443%\n",
      "Problem: A-n48-k7        Cost: 1118       Optimal Cost: 1073      \t Gap: 4.194%\n",
      "Problem: A-n39-k5        Cost: 846        Optimal Cost: 822       \t Gap: 2.920%\n",
      "Problem: A-n63-k10       Cost: 1331       Optimal Cost: 1314      \t Gap: 1.294%\n",
      "Problem: A-n36-k5        Cost: 817        Optimal Cost: 799       \t Gap: 2.253%\n",
      "Problem: A-n34-k5        Cost: 787        Optimal Cost: 778       \t Gap: 1.157%\n",
      "Problem: A-n55-k9        Cost: 1120       Optimal Cost: 1073      \t Gap: 4.380%\n",
      "Problem: A-n69-k9        Cost: 1187       Optimal Cost: 1159      \t Gap: 2.416%\n",
      "Problem: A-n61-k9        Cost: 1071       Optimal Cost: 1034      \t Gap: 3.578%\n",
      "Problem: A-n38-k5        Cost: 745        Optimal Cost: 730       \t Gap: 2.055%\n",
      "Problem: A-n53-k7        Cost: 1052       Optimal Cost: 1010      \t Gap: 4.158%\n",
      "Problem: A-n45-k6        Cost: 986        Optimal Cost: 944       \t Gap: 4.449%\n",
      "Problem: A-n80-k10       Cost: 1802       Optimal Cost: 1763      \t Gap: 2.212%\n",
      "Problem: A-n46-k7        Cost: 924        Optimal Cost: 914       \t Gap: 1.094%\n",
      "Problem: A-n33-k6        Cost: 751        Optimal Cost: 742       \t Gap: 1.213%\n",
      "Problem: A-n37-k5        Cost: 711        Optimal Cost: 669       \t Gap: 6.278%\n",
      "Problem: A-n60-k9        Cost: 1376       Optimal Cost: 1354      \t Gap: 1.625%\n",
      "Problem: A-n39-k6        Cost: 871        Optimal Cost: 831       \t Gap: 4.813%\n",
      "Problem: A-n32-k5        Cost: 804        Optimal Cost: 784       \t Gap: 2.551%\n",
      "Problem: A-n45-k7        Cost: 1156       Optimal Cost: 1146      \t Gap: 0.873%\n",
      "Problem: A-n65-k9        Cost: 1200       Optimal Cost: 1174      \t Gap: 2.215%\n",
      "Problem: A-n64-k9        Cost: 1434       Optimal Cost: 1401      \t Gap: 2.355%\n",
      "Problem: A-n62-k8        Cost: 1323       Optimal Cost: 1288      \t Gap: 2.717%\n",
      "Problem: A-n63-k9        Cost: 1649       Optimal Cost: 1616      \t Gap: 2.042%\n",
      "Problem: A-n54-k7        Cost: 1176       Optimal Cost: 1167      \t Gap: 0.771%\n",
      "Problem: B-n41-k6        Cost: 843        Optimal Cost: 829       \t Gap: 1.689%\n",
      "Problem: B-n35-k5        Cost: 975        Optimal Cost: 955       \t Gap: 2.094%\n",
      "Problem: B-n38-k6        Cost: 821        Optimal Cost: 805       \t Gap: 1.988%\n",
      "Problem: B-n64-k9        Cost: 928        Optimal Cost: 861       \t Gap: 7.782%\n",
      "Problem: B-n39-k5        Cost: 555        Optimal Cost: 549       \t Gap: 1.093%\n",
      "Problem: B-n50-k8        Cost: 1333       Optimal Cost: 1312      \t Gap: 1.601%\n",
      "Problem: B-n45-k6        Cost: 741        Optimal Cost: 678       \t Gap: 9.292%\n",
      "Problem: B-n43-k6        Cost: 752        Optimal Cost: 742       \t Gap: 1.348%\n",
      "Problem: B-n45-k5        Cost: 771        Optimal Cost: 751       \t Gap: 2.663%\n",
      "Problem: B-n34-k5        Cost: 797        Optimal Cost: 788       \t Gap: 1.142%\n",
      "Problem: B-n68-k9        Cost: 1296       Optimal Cost: 1272      \t Gap: 1.887%\n",
      "Problem: B-n56-k7        Cost: 726        Optimal Cost: 707       \t Gap: 2.687%\n",
      "Problem: B-n31-k5        Cost: 689        Optimal Cost: 672       \t Gap: 2.530%\n",
      "Problem: B-n44-k7        Cost: 940        Optimal Cost: 909       \t Gap: 3.410%\n",
      "Problem: B-n66-k9        Cost: 1342       Optimal Cost: 1316      \t Gap: 1.976%\n",
      "Problem: B-n57-k7        Cost: 1156       Optimal Cost: 1153      \t Gap: 0.260%\n",
      "Problem: B-n51-k7        Cost: 1025       Optimal Cost: 1032      \t Gap: -0.678%\n",
      "Problem: B-n50-k7        Cost: 762        Optimal Cost: 741       \t Gap: 2.834%\n",
      "Problem: B-n78-k10       Cost: 1275       Optimal Cost: 1221      \t Gap: 4.423%\n",
      "Problem: B-n63-k10       Cost: 1553       Optimal Cost: 1496      \t Gap: 3.810%\n",
      "Problem: B-n67-k10       Cost: 1059       Optimal Cost: 1032      \t Gap: 2.616%\n",
      "Problem: B-n52-k7        Cost: 765        Optimal Cost: 747       \t Gap: 2.410%\n",
      "Problem: B-n57-k9        Cost: 1607       Optimal Cost: 1598      \t Gap: 0.563%\n",
      "Problem: E-n101-k8       Cost: 856        Optimal Cost: 815       \t Gap: 5.031%\n",
      "Problem: E-n101-k14      Cost: 1126       Optimal Cost: 1067      \t Gap: 5.530%\n",
      "Problem: E-n23-k3        Cost: 574        Optimal Cost: 569       \t Gap: 0.879%\n",
      "Problem: E-n22-k4        Cost: 376        Optimal Cost: 375       \t Gap: 0.267%\n",
      "Problem: E-n76-k7        Cost: 708        Optimal Cost: 682       \t Gap: 3.812%\n",
      "Problem: E-n33-k4        Cost: 870        Optimal Cost: 835       \t Gap: 4.192%\n",
      "Skipping E-n13-k4 as it does not have node_coord\n",
      "Problem: E-n76-k14       Cost: 1052       Optimal Cost: 1021      \t Gap: 3.036%\n",
      "Problem: E-n76-k8        Cost: 761        Optimal Cost: 735       \t Gap: 3.537%\n",
      "Problem: E-n76-k10       Cost: 858        Optimal Cost: 830       \t Gap: 3.373%\n",
      "Skipping E-n31-k7 as it does not have node_coord\n",
      "Problem: E-n51-k5        Cost: 549        Optimal Cost: 521       \t Gap: 5.374%\n",
      "Problem: E-n30-k3        Cost: 519        Optimal Cost: 534       \t Gap: -2.809%\n",
      "Problem: F-n72-k4        Cost: 281        Optimal Cost: 237       \t Gap: 18.565%\n",
      "Problem: F-n135-k7       Cost: 1348       Optimal Cost: 1162      \t Gap: 16.007%\n",
      "Problem: F-n45-k4        Cost: 755        Optimal Cost: 724       \t Gap: 4.282%\n",
      "Problem: M-n200-k17      Cost: 1369       Optimal Cost: 1275      \t Gap: 7.373%\n",
      "Problem: M-n121-k7       Cost: 1068       Optimal Cost: 1034      \t Gap: 3.288%\n",
      "Problem: M-n200-k16      Cost: 1369       Optimal Cost: 1274      \t Gap: 7.457%\n",
      "Problem: M-n101-k10      Cost: 836        Optimal Cost: 820       \t Gap: 1.951%\n",
      "Problem: M-n151-k12      Cost: 1069       Optimal Cost: 1015      \t Gap: 5.320%\n",
      "Problem: P-n22-k2        Cost: 255        Optimal Cost: 216       \t Gap: 18.056%\n",
      "Problem: P-n76-k4        Cost: 635        Optimal Cost: 593       \t Gap: 7.083%\n",
      "Problem: P-n51-k10       Cost: 760        Optimal Cost: 741       \t Gap: 2.564%\n",
      "Problem: P-n55-k7        Cost: 596        Optimal Cost: 568       \t Gap: 4.930%\n",
      "Problem: P-n16-k8        Cost: 452        Optimal Cost: 450       \t Gap: 0.444%\n",
      "Problem: P-n19-k2        Cost: 233        Optimal Cost: 212       \t Gap: 9.906%\n",
      "Problem: P-n60-k15       Cost: 1004       Optimal Cost: 968       \t Gap: 3.719%\n",
      "Problem: P-n55-k8        Cost: 594        Optimal Cost: 588       \t Gap: 1.020%\n",
      "Problem: P-n22-k8        Cost: 596        Optimal Cost: 603       \t Gap: -1.161%\n",
      "Problem: P-n40-k5        Cost: 488        Optimal Cost: 458       \t Gap: 6.550%\n",
      "Problem: P-n20-k2        Cost: 233        Optimal Cost: 216       \t Gap: 7.870%\n",
      "Problem: P-n55-k15       Cost: 984        Optimal Cost: 989       \t Gap: -0.506%\n",
      "Problem: P-n65-k10       Cost: 808        Optimal Cost: 792       \t Gap: 2.020%\n",
      "Problem: P-n101-k4       Cost: 724        Optimal Cost: 681       \t Gap: 6.314%\n",
      "Problem: P-n21-k2        Cost: 232        Optimal Cost: 211       \t Gap: 9.953%\n",
      "Problem: P-n50-k8        Cost: 658        Optimal Cost: 631       \t Gap: 4.279%\n",
      "Problem: P-n23-k8        Cost: 544        Optimal Cost: 529       \t Gap: 2.836%\n",
      "Problem: P-n76-k5        Cost: 659        Optimal Cost: 627       \t Gap: 5.104%\n",
      "Problem: P-n70-k10       Cost: 850        Optimal Cost: 827       \t Gap: 2.781%\n",
      "Problem: P-n50-k7        Cost: 569        Optimal Cost: 554       \t Gap: 2.708%\n",
      "Problem: P-n50-k10       Cost: 727        Optimal Cost: 696       \t Gap: 4.454%\n",
      "Problem: P-n45-k5        Cost: 526        Optimal Cost: 510       \t Gap: 3.137%\n",
      "Problem: P-n55-k10       Cost: 712        Optimal Cost: 694       \t Gap: 2.594%\n",
      "Problem: P-n60-k10       Cost: 767        Optimal Cost: 744       \t Gap: 3.091%\n",
      "Problem: X-n120-k6       Cost: 13765      Optimal Cost: 13332     \t Gap: 3.248%\n",
      "Problem: X-n856-k95      Cost: 98393      Optimal Cost: 88965     \t Gap: 10.597%\n",
      "Problem: X-n766-k71      Cost: 130052     Optimal Cost: 114417    \t Gap: 13.665%\n",
      "Problem: X-n524-k153     Cost: 174075     Optimal Cost: 154593    \t Gap: 12.602%\n",
      "Problem: X-n420-k130     Cost: 116763     Optimal Cost: 107798    \t Gap: 8.316%\n",
      "Problem: X-n561-k42      Cost: 49455      Optimal Cost: 42717     \t Gap: 15.774%\n",
      "Problem: X-n214-k11      Cost: 11670      Optimal Cost: 10856     \t Gap: 7.498%\n",
      "Problem: X-n701-k44      Cost: 90970      Optimal Cost: 81923     \t Gap: 11.043%\n",
      "Problem: X-n256-k16      Cost: 19998      Optimal Cost: 18839     \t Gap: 6.152%\n",
      "Problem: X-n125-k30      Cost: 58522      Optimal Cost: 55539     \t Gap: 5.371%\n",
      "Problem: X-n261-k13      Cost: 28510      Optimal Cost: 26558     \t Gap: 7.350%\n",
      "Problem: X-n895-k37      Cost: 64525      Optimal Cost: 53860     \t Gap: 19.801%\n",
      "Problem: X-n819-k171     Cost: 174609     Optimal Cost: 158121    \t Gap: 10.427%\n",
      "Problem: X-n513-k21      Cost: 28566      Optimal Cost: 24201     \t Gap: 18.036%\n",
      "Problem: X-n536-k96      Cost: 103337     Optimal Cost: 94846     \t Gap: 8.952%\n",
      "Problem: X-n344-k43      Cost: 44857      Optimal Cost: 42050     \t Gap: 6.675%\n",
      "Problem: X-n1001-k43     Cost: 85998      Optimal Cost: 72355     \t Gap: 18.856%\n",
      "Problem: X-n439-k37      Cost: 40029      Optimal Cost: 36391     \t Gap: 9.997%\n",
      "Problem: X-n586-k159     Cost: 205770     Optimal Cost: 190316    \t Gap: 8.120%\n",
      "Problem: X-n401-k29      Cost: 69381      Optimal Cost: 66154     \t Gap: 4.878%\n",
      "Problem: X-n641-k35      Cost: 70676      Optimal Cost: 63684     \t Gap: 10.979%\n",
      "Problem: X-n783-k48      Cost: 83165      Optimal Cost: 72386     \t Gap: 14.891%\n",
      "Problem: X-n480-k70      Cost: 95028      Optimal Cost: 89449     \t Gap: 6.237%\n",
      "Problem: X-n251-k28      Cost: 40399      Optimal Cost: 38684     \t Gap: 4.433%\n",
      "Problem: X-n237-k14      Cost: 29595      Optimal Cost: 27042     \t Gap: 9.441%\n",
      "Problem: X-n613-k62      Cost: 66803      Optimal Cost: 59535     \t Gap: 12.208%\n",
      "Problem: X-n228-k23      Cost: 28798      Optimal Cost: 25742     \t Gap: 11.872%\n",
      "Problem: X-n101-k25      Cost: 29035      Optimal Cost: 27591     \t Gap: 5.234%\n",
      "Problem: X-n336-k84      Cost: 146486     Optimal Cost: 139111    \t Gap: 5.302%\n",
      "Problem: X-n233-k16      Cost: 20758      Optimal Cost: 19230     \t Gap: 7.946%\n",
      "Problem: X-n266-k58      Cost: 79816      Optimal Cost: 75478     \t Gap: 5.747%\n",
      "Problem: X-n294-k50      Cost: 50559      Optimal Cost: 47161     \t Gap: 7.205%\n",
      "Problem: X-n209-k16      Cost: 31876      Optimal Cost: 30656     \t Gap: 3.980%\n",
      "Problem: X-n393-k38      Cost: 41669      Optimal Cost: 38260     \t Gap: 8.910%\n",
      "Problem: X-n469-k138     Cost: 238481     Optimal Cost: 221824    \t Gap: 7.509%\n",
      "Problem: X-n351-k40      Cost: 28341      Optimal Cost: 25896     \t Gap: 9.442%\n",
      "Problem: X-n139-k10      Cost: 13812      Optimal Cost: 13590     \t Gap: 1.634%\n",
      "Problem: X-n289-k60      Cost: 100443     Optimal Cost: 95151     \t Gap: 5.562%\n",
      "Problem: X-n270-k35      Cost: 37384      Optimal Cost: 35291     \t Gap: 5.931%\n",
      "Problem: X-n685-k75      Cost: 77687      Optimal Cost: 68205     \t Gap: 13.902%\n",
      "Problem: X-n599-k92      Cost: 116603     Optimal Cost: 108451    \t Gap: 7.517%\n",
      "Problem: X-n749-k98      Cost: 85048      Optimal Cost: 77269     \t Gap: 10.067%\n",
      "Problem: X-n134-k13      Cost: 11585      Optimal Cost: 10916     \t Gap: 6.129%\n",
      "Problem: X-n548-k50      Cost: 100914     Optimal Cost: 86700     \t Gap: 16.394%\n",
      "Problem: X-n223-k34      Cost: 42251      Optimal Cost: 40437     \t Gap: 4.486%\n",
      "Problem: X-n502-k39      Cost: 71836      Optimal Cost: 69226     \t Gap: 3.770%\n",
      "Problem: X-n106-k14      Cost: 27150      Optimal Cost: 26362     \t Gap: 2.989%\n",
      "Problem: X-n359-k29      Cost: 55243      Optimal Cost: 51505     \t Gap: 7.258%\n",
      "Problem: X-n162-k11      Cost: 14664      Optimal Cost: 14138     \t Gap: 3.720%\n",
      "Problem: X-n115-k10      Cost: 13338      Optimal Cost: 12747     \t Gap: 4.636%\n",
      "Problem: X-n655-k131     Cost: 112067     Optimal Cost: 106780    \t Gap: 4.951%\n",
      "Problem: X-n936-k151     Cost: 163073     Optimal Cost: 132715    \t Gap: 22.875%\n",
      "Problem: X-n573-k30      Cost: 55797      Optimal Cost: 50673     \t Gap: 10.112%\n",
      "Problem: X-n176-k26      Cost: 51400      Optimal Cost: 47812     \t Gap: 7.504%\n",
      "Problem: X-n186-k15      Cost: 25140      Optimal Cost: 24145     \t Gap: 4.121%\n",
      "Problem: X-n376-k94      Cost: 151981     Optimal Cost: 147713    \t Gap: 2.889%\n",
      "Problem: X-n153-k22      Cost: 23478      Optimal Cost: 21220     \t Gap: 10.641%\n",
      "Problem: X-n303-k21      Cost: 23483      Optimal Cost: 21736     \t Gap: 8.037%\n",
      "Problem: X-n167-k10      Cost: 21412      Optimal Cost: 20557     \t Gap: 4.159%\n",
      "Problem: X-n322-k28      Cost: 32603      Optimal Cost: 29834     \t Gap: 9.281%\n",
      "Problem: X-n110-k13      Cost: 15314      Optimal Cost: 14971     \t Gap: 2.291%\n",
      "Problem: X-n449-k29      Cost: 60634      Optimal Cost: 55233     \t Gap: 9.779%\n",
      "Problem: X-n317-k53      Cost: 80714      Optimal Cost: 78355     \t Gap: 3.011%\n",
      "Problem: X-n200-k36      Cost: 61199      Optimal Cost: 58578     \t Gap: 4.474%\n",
      "Problem: X-n143-k7       Cost: 16257      Optimal Cost: 15700     \t Gap: 3.548%\n",
      "Problem: X-n733-k159     Cost: 148786     Optimal Cost: 136187    \t Gap: 9.251%\n",
      "Problem: X-n429-k61      Cost: 70426      Optimal Cost: 65449     \t Gap: 7.604%\n",
      "Problem: X-n157-k13      Cost: 17339      Optimal Cost: 16876     \t Gap: 2.744%\n",
      "Problem: X-n181-k23      Cost: 26097      Optimal Cost: 25569     \t Gap: 2.065%\n",
      "Problem: X-n491-k59      Cost: 72518      Optimal Cost: 66483     \t Gap: 9.078%\n",
      "Problem: X-n204-k19      Cost: 20608      Optimal Cost: 19565     \t Gap: 5.331%\n",
      "Problem: X-n331-k15      Cost: 33966      Optimal Cost: 31102     \t Gap: 9.208%\n",
      "Problem: X-n801-k40      Cost: 86024      Optimal Cost: 73311     \t Gap: 17.341%\n",
      "Problem: X-n172-k51      Cost: 48118      Optimal Cost: 45607     \t Gap: 5.506%\n",
      "Problem: X-n837-k142     Cost: 208252     Optimal Cost: 193737    \t Gap: 7.492%\n",
      "Problem: X-n195-k51      Cost: 47390      Optimal Cost: 44225     \t Gap: 7.157%\n",
      "Problem: X-n670-k130     Cost: 169056     Optimal Cost: 146332    \t Gap: 15.529%\n",
      "Problem: X-n327-k20      Cost: 29784      Optimal Cost: 27532     \t Gap: 8.180%\n",
      "Problem: X-n627-k43      Cost: 67339      Optimal Cost: 62164     \t Gap: 8.325%\n",
      "Problem: X-n298-k31      Cost: 36706      Optimal Cost: 34231     \t Gap: 7.230%\n",
      "Problem: X-n284-k15      Cost: 22043      Optimal Cost: 20226     \t Gap: 8.983%\n",
      "Problem: X-n190-k8       Cost: 17892      Optimal Cost: 16980     \t Gap: 5.371%\n",
      "Problem: X-n148-k46      Cost: 45036      Optimal Cost: 43448     \t Gap: 3.655%\n",
      "Problem: X-n876-k59      Cost: 107229     Optimal Cost: 99299     \t Gap: 7.986%\n",
      "Problem: X-n313-k71      Cost: 99661      Optimal Cost: 94043     \t Gap: 5.974%\n",
      "Problem: X-n280-k17      Cost: 36549      Optimal Cost: 33503     \t Gap: 9.092%\n",
      "Problem: X-n275-k28      Cost: 24187      Optimal Cost: 21245     \t Gap: 13.848%\n",
      "Problem: X-n247-k50      Cost: 40639      Optimal Cost: 37274     \t Gap: 9.028%\n",
      "Problem: X-n219-k73      Cost: 120348     Optimal Cost: 117595    \t Gap: 2.341%\n",
      "Problem: X-n129-k18      Cost: 29598      Optimal Cost: 28940     \t Gap: 2.274%\n",
      "Problem: X-n242-k48      Cost: 85704      Optimal Cost: 82751     \t Gap: 3.569%\n",
      "Problem: X-n411-k19      Cost: 22868      Optimal Cost: 19712     \t Gap: 16.011%\n",
      "Problem: X-n367-k17      Cost: 25578      Optimal Cost: 22814     \t Gap: 12.115%\n",
      "Problem: X-n459-k26      Cost: 27341      Optimal Cost: 24139     \t Gap: 13.265%\n",
      "Problem: X-n979-k58      Cost: 129770     Optimal Cost: 118976    \t Gap: 9.072%\n",
      "Problem: X-n716-k35      Cost: 49709      Optimal Cost: 43373     \t Gap: 14.608%\n",
      "Problem: X-n957-k87      Cost: 102964     Optimal Cost: 85465     \t Gap: 20.475%\n",
      "Problem: X-n384-k52      Cost: 70231      Optimal Cost: 65940     \t Gap: 6.507%\n",
      "Problem: X-n916-k207     Cost: 352732     Optimal Cost: 329179    \t Gap: 7.155%\n",
      "Problem: X-n308-k13      Cost: 28434      Optimal Cost: 25859     \t Gap: 9.958%\n"
     ]
    }
   ],
   "source": [
    "from tensordict import TensorDict\n",
    "from math import ceil\n",
    "import time\n",
    "import vrplib\n",
    "\n",
    "\n",
    "model.eval().to(device)\n",
    "\n",
    "results = []\n",
    "\n",
    "for instance in instances:\n",
    "    problem = vrplib.read_instance(instances[instance][\"data\"])\n",
    "\n",
    "    if problem.get(\"node_coord\", None) is None:\n",
    "        print(f\"Skipping {instance} as it does not have node_coord\")\n",
    "        continue\n",
    "    coords = torch.tensor(problem['node_coord']).float()\n",
    "    coords_norm = normalize_coord(coords)\n",
    "\n",
    "    original_capacity = problem['capacity']\n",
    "    demand = torch.tensor(problem['demand'][1:]).float() / original_capacity\n",
    "    original_capacity = torch.tensor(original_capacity)[None]         \n",
    "\n",
    "    # Make instance\n",
    "    td_instance = TensorDict({\n",
    "        \"locs\": coords_norm,\n",
    "        \"demand_linehaul\": demand,\n",
    "        \"capacity_original\": original_capacity,\n",
    "    },\n",
    "    batch_size=[])[None]\n",
    "\n",
    "    td_reset = env.reset(td_instance).to(device)\n",
    "    \n",
    "    start = time.time()\n",
    "    actions = evaluate(model, td_reset.clone(), env)[\"best_aug_actions\"]\n",
    "    inference_time = time.time() - start        \n",
    "\n",
    "    # Obtain reward from the environment with new locs\n",
    "    td_reset[\"locs\"] = coords[None] # unnormalized\n",
    "    reward = env.get_reward(td_reset, actions)\n",
    "    \n",
    "    # Load the optimal cost\n",
    "    solution = vrplib.read_solution(instances[instance][\"solution\"])\n",
    "    optimal_cost = solution['cost'] # note that this cost is somehow slightly lower than the one calculated from the distance matrix\n",
    "    \n",
    "    # Calculate the gap and print\n",
    "    cost = ceil(-reward.item())\n",
    "    gap = (cost - optimal_cost) / optimal_cost\n",
    "    print(f'Problem: {instance:<15} Cost: {cost:<10} Optimal Cost: {optimal_cost:<10}\\t Gap: {gap:.3%}')\n",
    "    \n",
    "    results.append({\n",
    "        \"instance\": instance,\n",
    "        \"cost\": cost,\n",
    "        \"optimal_cost\": optimal_cost,\n",
    "        \"gap\": gap,\n",
    "        \"inference_time\": inference_time,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the results with pkl\n",
    "import pickle\n",
    "import os \n",
    "\n",
    "SAVEDIR = \"results/cvrplib/\"\n",
    "os.makedirs(SAVEDIR, exist_ok=True)\n",
    "\n",
    "# TODO: change the filename\n",
    "with open(SAVEDIR+'rf-transformer.pkl', 'wb') as f:\n",
    "    pickle.dump(results, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Problem: A-n32-k5        Cost: 804        Optimal Cost: 784       \t Gap: 2.551%\n",
      "Problem: A-n33-k5        Cost: 694        Optimal Cost: 661       \t Gap: 4.992%\n",
      "Problem: A-n33-k6        Cost: 751        Optimal Cost: 742       \t Gap: 1.213%\n",
      "Problem: A-n34-k5        Cost: 787        Optimal Cost: 778       \t Gap: 1.157%\n",
      "Problem: A-n36-k5        Cost: 817        Optimal Cost: 799       \t Gap: 2.253%\n",
      "Problem: A-n37-k5        Cost: 711        Optimal Cost: 669       \t Gap: 6.278%\n",
      "Problem: A-n37-k6        Cost: 970        Optimal Cost: 949       \t Gap: 2.213%\n",
      "Problem: A-n38-k5        Cost: 745        Optimal Cost: 730       \t Gap: 2.055%\n",
      "Problem: A-n39-k5        Cost: 846        Optimal Cost: 822       \t Gap: 2.920%\n",
      "Problem: A-n39-k6        Cost: 871        Optimal Cost: 831       \t Gap: 4.813%\n",
      "Problem: A-n44-k6        Cost: 988        Optimal Cost: 937       \t Gap: 5.443%\n",
      "Problem: A-n45-k6        Cost: 986        Optimal Cost: 944       \t Gap: 4.449%\n",
      "Problem: A-n45-k7        Cost: 1156       Optimal Cost: 1146      \t Gap: 0.873%\n",
      "Problem: A-n46-k7        Cost: 924        Optimal Cost: 914       \t Gap: 1.094%\n",
      "Problem: A-n48-k7        Cost: 1118       Optimal Cost: 1073      \t Gap: 4.194%\n",
      "Problem: A-n53-k7        Cost: 1052       Optimal Cost: 1010      \t Gap: 4.158%\n",
      "Problem: A-n54-k7        Cost: 1176       Optimal Cost: 1167      \t Gap: 0.771%\n",
      "Problem: A-n55-k9        Cost: 1120       Optimal Cost: 1073      \t Gap: 4.380%\n",
      "Problem: A-n60-k9        Cost: 1376       Optimal Cost: 1354      \t Gap: 1.625%\n",
      "Problem: A-n61-k9        Cost: 1071       Optimal Cost: 1034      \t Gap: 3.578%\n",
      "Problem: A-n62-k8        Cost: 1323       Optimal Cost: 1288      \t Gap: 2.717%\n",
      "Problem: A-n63-k10       Cost: 1331       Optimal Cost: 1314      \t Gap: 1.294%\n",
      "Problem: A-n63-k9        Cost: 1649       Optimal Cost: 1616      \t Gap: 2.042%\n",
      "Problem: A-n64-k9        Cost: 1434       Optimal Cost: 1401      \t Gap: 2.355%\n",
      "Problem: A-n65-k9        Cost: 1200       Optimal Cost: 1174      \t Gap: 2.215%\n",
      "Problem: A-n69-k9        Cost: 1187       Optimal Cost: 1159      \t Gap: 2.416%\n",
      "Problem: A-n80-k10       Cost: 1802       Optimal Cost: 1763      \t Gap: 2.212%\n",
      "Problem: B-n31-k5        Cost: 689        Optimal Cost: 672       \t Gap: 2.530%\n",
      "Problem: B-n34-k5        Cost: 797        Optimal Cost: 788       \t Gap: 1.142%\n",
      "Problem: B-n35-k5        Cost: 975        Optimal Cost: 955       \t Gap: 2.094%\n",
      "Problem: B-n38-k6        Cost: 821        Optimal Cost: 805       \t Gap: 1.988%\n",
      "Problem: B-n39-k5        Cost: 555        Optimal Cost: 549       \t Gap: 1.093%\n",
      "Problem: B-n41-k6        Cost: 843        Optimal Cost: 829       \t Gap: 1.689%\n",
      "Problem: B-n43-k6        Cost: 752        Optimal Cost: 742       \t Gap: 1.348%\n",
      "Problem: B-n44-k7        Cost: 940        Optimal Cost: 909       \t Gap: 3.410%\n",
      "Problem: B-n45-k5        Cost: 771        Optimal Cost: 751       \t Gap: 2.663%\n",
      "Problem: B-n45-k6        Cost: 741        Optimal Cost: 678       \t Gap: 9.292%\n",
      "Problem: B-n50-k7        Cost: 762        Optimal Cost: 741       \t Gap: 2.834%\n",
      "Problem: B-n50-k8        Cost: 1333       Optimal Cost: 1312      \t Gap: 1.601%\n",
      "Problem: B-n51-k7        Cost: 1025       Optimal Cost: 1032      \t Gap: -0.678%\n",
      "Problem: B-n52-k7        Cost: 765        Optimal Cost: 747       \t Gap: 2.410%\n",
      "Problem: B-n56-k7        Cost: 726        Optimal Cost: 707       \t Gap: 2.687%\n",
      "Problem: B-n57-k7        Cost: 1156       Optimal Cost: 1153      \t Gap: 0.260%\n",
      "Problem: B-n57-k9        Cost: 1607       Optimal Cost: 1598      \t Gap: 0.563%\n",
      "Problem: B-n63-k10       Cost: 1553       Optimal Cost: 1496      \t Gap: 3.810%\n",
      "Problem: B-n64-k9        Cost: 928        Optimal Cost: 861       \t Gap: 7.782%\n",
      "Problem: B-n66-k9        Cost: 1342       Optimal Cost: 1316      \t Gap: 1.976%\n",
      "Problem: B-n67-k10       Cost: 1059       Optimal Cost: 1032      \t Gap: 2.616%\n",
      "Problem: B-n68-k9        Cost: 1296       Optimal Cost: 1272      \t Gap: 1.887%\n",
      "Problem: B-n78-k10       Cost: 1275       Optimal Cost: 1221      \t Gap: 4.423%\n",
      "Problem: E-n101-k14      Cost: 1126       Optimal Cost: 1067      \t Gap: 5.530%\n",
      "Problem: E-n101-k8       Cost: 856        Optimal Cost: 815       \t Gap: 5.031%\n",
      "Problem: E-n22-k4        Cost: 376        Optimal Cost: 375       \t Gap: 0.267%\n",
      "Problem: E-n23-k3        Cost: 574        Optimal Cost: 569       \t Gap: 0.879%\n",
      "Problem: E-n30-k3        Cost: 519        Optimal Cost: 534       \t Gap: -2.809%\n",
      "Problem: E-n33-k4        Cost: 870        Optimal Cost: 835       \t Gap: 4.192%\n",
      "Problem: E-n51-k5        Cost: 549        Optimal Cost: 521       \t Gap: 5.374%\n",
      "Problem: E-n76-k10       Cost: 858        Optimal Cost: 830       \t Gap: 3.373%\n",
      "Problem: E-n76-k14       Cost: 1052       Optimal Cost: 1021      \t Gap: 3.036%\n",
      "Problem: E-n76-k7        Cost: 708        Optimal Cost: 682       \t Gap: 3.812%\n",
      "Problem: E-n76-k8        Cost: 761        Optimal Cost: 735       \t Gap: 3.537%\n",
      "Problem: F-n135-k7       Cost: 1348       Optimal Cost: 1162      \t Gap: 16.007%\n",
      "Problem: F-n45-k4        Cost: 755        Optimal Cost: 724       \t Gap: 4.282%\n",
      "Problem: F-n72-k4        Cost: 281        Optimal Cost: 237       \t Gap: 18.565%\n",
      "Problem: M-n101-k10      Cost: 836        Optimal Cost: 820       \t Gap: 1.951%\n",
      "Problem: M-n121-k7       Cost: 1068       Optimal Cost: 1034      \t Gap: 3.288%\n",
      "Problem: M-n151-k12      Cost: 1069       Optimal Cost: 1015      \t Gap: 5.320%\n",
      "Problem: M-n200-k16      Cost: 1369       Optimal Cost: 1274      \t Gap: 7.457%\n",
      "Problem: M-n200-k17      Cost: 1369       Optimal Cost: 1275      \t Gap: 7.373%\n",
      "Problem: P-n101-k4       Cost: 724        Optimal Cost: 681       \t Gap: 6.314%\n",
      "Problem: P-n16-k8        Cost: 452        Optimal Cost: 450       \t Gap: 0.444%\n",
      "Problem: P-n19-k2        Cost: 233        Optimal Cost: 212       \t Gap: 9.906%\n",
      "Problem: P-n20-k2        Cost: 233        Optimal Cost: 216       \t Gap: 7.870%\n",
      "Problem: P-n21-k2        Cost: 232        Optimal Cost: 211       \t Gap: 9.953%\n",
      "Problem: P-n22-k2        Cost: 255        Optimal Cost: 216       \t Gap: 18.056%\n",
      "Problem: P-n22-k8        Cost: 596        Optimal Cost: 603       \t Gap: -1.161%\n",
      "Problem: P-n23-k8        Cost: 544        Optimal Cost: 529       \t Gap: 2.836%\n",
      "Problem: P-n40-k5        Cost: 488        Optimal Cost: 458       \t Gap: 6.550%\n",
      "Problem: P-n45-k5        Cost: 526        Optimal Cost: 510       \t Gap: 3.137%\n",
      "Problem: P-n50-k10       Cost: 727        Optimal Cost: 696       \t Gap: 4.454%\n",
      "Problem: P-n50-k7        Cost: 569        Optimal Cost: 554       \t Gap: 2.708%\n",
      "Problem: P-n50-k8        Cost: 658        Optimal Cost: 631       \t Gap: 4.279%\n",
      "Problem: P-n51-k10       Cost: 760        Optimal Cost: 741       \t Gap: 2.564%\n",
      "Problem: P-n55-k10       Cost: 712        Optimal Cost: 694       \t Gap: 2.594%\n",
      "Problem: P-n55-k15       Cost: 984        Optimal Cost: 989       \t Gap: -0.506%\n",
      "Problem: P-n55-k7        Cost: 596        Optimal Cost: 568       \t Gap: 4.930%\n",
      "Problem: P-n55-k8        Cost: 594        Optimal Cost: 588       \t Gap: 1.020%\n",
      "Problem: P-n60-k10       Cost: 767        Optimal Cost: 744       \t Gap: 3.091%\n",
      "Problem: P-n60-k15       Cost: 1004       Optimal Cost: 968       \t Gap: 3.719%\n",
      "Problem: P-n65-k10       Cost: 808        Optimal Cost: 792       \t Gap: 2.020%\n",
      "Problem: P-n70-k10       Cost: 850        Optimal Cost: 827       \t Gap: 2.781%\n",
      "Problem: P-n76-k4        Cost: 635        Optimal Cost: 593       \t Gap: 7.083%\n",
      "Problem: P-n76-k5        Cost: 659        Optimal Cost: 627       \t Gap: 5.104%\n",
      "Problem: X-n1001-k43     Cost: 85998      Optimal Cost: 72355     \t Gap: 18.856%\n",
      "Problem: X-n101-k25      Cost: 29035      Optimal Cost: 27591     \t Gap: 5.234%\n",
      "Problem: X-n106-k14      Cost: 27150      Optimal Cost: 26362     \t Gap: 2.989%\n",
      "Problem: X-n110-k13      Cost: 15314      Optimal Cost: 14971     \t Gap: 2.291%\n",
      "Problem: X-n115-k10      Cost: 13338      Optimal Cost: 12747     \t Gap: 4.636%\n",
      "Problem: X-n120-k6       Cost: 13765      Optimal Cost: 13332     \t Gap: 3.248%\n",
      "Problem: X-n125-k30      Cost: 58522      Optimal Cost: 55539     \t Gap: 5.371%\n",
      "Problem: X-n129-k18      Cost: 29598      Optimal Cost: 28940     \t Gap: 2.274%\n",
      "Problem: X-n134-k13      Cost: 11585      Optimal Cost: 10916     \t Gap: 6.129%\n",
      "Problem: X-n139-k10      Cost: 13812      Optimal Cost: 13590     \t Gap: 1.634%\n",
      "Problem: X-n143-k7       Cost: 16257      Optimal Cost: 15700     \t Gap: 3.548%\n",
      "Problem: X-n148-k46      Cost: 45036      Optimal Cost: 43448     \t Gap: 3.655%\n",
      "Problem: X-n153-k22      Cost: 23478      Optimal Cost: 21220     \t Gap: 10.641%\n",
      "Problem: X-n157-k13      Cost: 17339      Optimal Cost: 16876     \t Gap: 2.744%\n",
      "Problem: X-n162-k11      Cost: 14664      Optimal Cost: 14138     \t Gap: 3.720%\n",
      "Problem: X-n167-k10      Cost: 21412      Optimal Cost: 20557     \t Gap: 4.159%\n",
      "Problem: X-n172-k51      Cost: 48118      Optimal Cost: 45607     \t Gap: 5.506%\n",
      "Problem: X-n176-k26      Cost: 51400      Optimal Cost: 47812     \t Gap: 7.504%\n",
      "Problem: X-n181-k23      Cost: 26097      Optimal Cost: 25569     \t Gap: 2.065%\n",
      "Problem: X-n186-k15      Cost: 25140      Optimal Cost: 24145     \t Gap: 4.121%\n",
      "Problem: X-n190-k8       Cost: 17892      Optimal Cost: 16980     \t Gap: 5.371%\n",
      "Problem: X-n195-k51      Cost: 47390      Optimal Cost: 44225     \t Gap: 7.157%\n",
      "Problem: X-n200-k36      Cost: 61199      Optimal Cost: 58578     \t Gap: 4.474%\n",
      "Problem: X-n204-k19      Cost: 20608      Optimal Cost: 19565     \t Gap: 5.331%\n",
      "Problem: X-n209-k16      Cost: 31876      Optimal Cost: 30656     \t Gap: 3.980%\n",
      "Problem: X-n214-k11      Cost: 11670      Optimal Cost: 10856     \t Gap: 7.498%\n",
      "Problem: X-n219-k73      Cost: 120348     Optimal Cost: 117595    \t Gap: 2.341%\n",
      "Problem: X-n223-k34      Cost: 42251      Optimal Cost: 40437     \t Gap: 4.486%\n",
      "Problem: X-n228-k23      Cost: 28798      Optimal Cost: 25742     \t Gap: 11.872%\n",
      "Problem: X-n233-k16      Cost: 20758      Optimal Cost: 19230     \t Gap: 7.946%\n",
      "Problem: X-n237-k14      Cost: 29595      Optimal Cost: 27042     \t Gap: 9.441%\n",
      "Problem: X-n242-k48      Cost: 85704      Optimal Cost: 82751     \t Gap: 3.569%\n",
      "Problem: X-n247-k50      Cost: 40639      Optimal Cost: 37274     \t Gap: 9.028%\n",
      "Problem: X-n251-k28      Cost: 40399      Optimal Cost: 38684     \t Gap: 4.433%\n",
      "Problem: X-n256-k16      Cost: 19998      Optimal Cost: 18839     \t Gap: 6.152%\n",
      "Problem: X-n261-k13      Cost: 28510      Optimal Cost: 26558     \t Gap: 7.350%\n",
      "Problem: X-n266-k58      Cost: 79816      Optimal Cost: 75478     \t Gap: 5.747%\n",
      "Problem: X-n270-k35      Cost: 37384      Optimal Cost: 35291     \t Gap: 5.931%\n",
      "Problem: X-n275-k28      Cost: 24187      Optimal Cost: 21245     \t Gap: 13.848%\n",
      "Problem: X-n280-k17      Cost: 36549      Optimal Cost: 33503     \t Gap: 9.092%\n",
      "Problem: X-n284-k15      Cost: 22043      Optimal Cost: 20226     \t Gap: 8.983%\n",
      "Problem: X-n289-k60      Cost: 100443     Optimal Cost: 95151     \t Gap: 5.562%\n",
      "Problem: X-n294-k50      Cost: 50559      Optimal Cost: 47161     \t Gap: 7.205%\n",
      "Problem: X-n298-k31      Cost: 36706      Optimal Cost: 34231     \t Gap: 7.230%\n",
      "Problem: X-n303-k21      Cost: 23483      Optimal Cost: 21736     \t Gap: 8.037%\n",
      "Problem: X-n308-k13      Cost: 28434      Optimal Cost: 25859     \t Gap: 9.958%\n",
      "Problem: X-n313-k71      Cost: 99661      Optimal Cost: 94043     \t Gap: 5.974%\n",
      "Problem: X-n317-k53      Cost: 80714      Optimal Cost: 78355     \t Gap: 3.011%\n",
      "Problem: X-n322-k28      Cost: 32603      Optimal Cost: 29834     \t Gap: 9.281%\n",
      "Problem: X-n327-k20      Cost: 29784      Optimal Cost: 27532     \t Gap: 8.180%\n",
      "Problem: X-n331-k15      Cost: 33966      Optimal Cost: 31102     \t Gap: 9.208%\n",
      "Problem: X-n336-k84      Cost: 146486     Optimal Cost: 139111    \t Gap: 5.302%\n",
      "Problem: X-n344-k43      Cost: 44857      Optimal Cost: 42050     \t Gap: 6.675%\n",
      "Problem: X-n351-k40      Cost: 28341      Optimal Cost: 25896     \t Gap: 9.442%\n",
      "Problem: X-n359-k29      Cost: 55243      Optimal Cost: 51505     \t Gap: 7.258%\n",
      "Problem: X-n367-k17      Cost: 25578      Optimal Cost: 22814     \t Gap: 12.115%\n",
      "Problem: X-n376-k94      Cost: 151981     Optimal Cost: 147713    \t Gap: 2.889%\n",
      "Problem: X-n384-k52      Cost: 70231      Optimal Cost: 65940     \t Gap: 6.507%\n",
      "Problem: X-n393-k38      Cost: 41669      Optimal Cost: 38260     \t Gap: 8.910%\n",
      "Problem: X-n401-k29      Cost: 69381      Optimal Cost: 66154     \t Gap: 4.878%\n",
      "Problem: X-n411-k19      Cost: 22868      Optimal Cost: 19712     \t Gap: 16.011%\n",
      "Problem: X-n420-k130     Cost: 116763     Optimal Cost: 107798    \t Gap: 8.316%\n",
      "Problem: X-n429-k61      Cost: 70426      Optimal Cost: 65449     \t Gap: 7.604%\n",
      "Problem: X-n439-k37      Cost: 40029      Optimal Cost: 36391     \t Gap: 9.997%\n",
      "Problem: X-n449-k29      Cost: 60634      Optimal Cost: 55233     \t Gap: 9.779%\n",
      "Problem: X-n459-k26      Cost: 27341      Optimal Cost: 24139     \t Gap: 13.265%\n",
      "Problem: X-n469-k138     Cost: 238481     Optimal Cost: 221824    \t Gap: 7.509%\n",
      "Problem: X-n480-k70      Cost: 95028      Optimal Cost: 89449     \t Gap: 6.237%\n",
      "Problem: X-n491-k59      Cost: 72518      Optimal Cost: 66483     \t Gap: 9.078%\n",
      "Problem: X-n502-k39      Cost: 71836      Optimal Cost: 69226     \t Gap: 3.770%\n",
      "Problem: X-n513-k21      Cost: 28566      Optimal Cost: 24201     \t Gap: 18.036%\n",
      "Problem: X-n524-k153     Cost: 174075     Optimal Cost: 154593    \t Gap: 12.602%\n",
      "Problem: X-n536-k96      Cost: 103337     Optimal Cost: 94846     \t Gap: 8.952%\n",
      "Problem: X-n548-k50      Cost: 100914     Optimal Cost: 86700     \t Gap: 16.394%\n",
      "Problem: X-n561-k42      Cost: 49455      Optimal Cost: 42717     \t Gap: 15.774%\n",
      "Problem: X-n573-k30      Cost: 55797      Optimal Cost: 50673     \t Gap: 10.112%\n",
      "Problem: X-n586-k159     Cost: 205770     Optimal Cost: 190316    \t Gap: 8.120%\n",
      "Problem: X-n599-k92      Cost: 116603     Optimal Cost: 108451    \t Gap: 7.517%\n",
      "Problem: X-n613-k62      Cost: 66803      Optimal Cost: 59535     \t Gap: 12.208%\n",
      "Problem: X-n627-k43      Cost: 67339      Optimal Cost: 62164     \t Gap: 8.325%\n",
      "Problem: X-n641-k35      Cost: 70676      Optimal Cost: 63684     \t Gap: 10.979%\n",
      "Problem: X-n655-k131     Cost: 112067     Optimal Cost: 106780    \t Gap: 4.951%\n",
      "Problem: X-n670-k130     Cost: 169056     Optimal Cost: 146332    \t Gap: 15.529%\n",
      "Problem: X-n685-k75      Cost: 77687      Optimal Cost: 68205     \t Gap: 13.902%\n",
      "Problem: X-n701-k44      Cost: 90970      Optimal Cost: 81923     \t Gap: 11.043%\n",
      "Problem: X-n716-k35      Cost: 49709      Optimal Cost: 43373     \t Gap: 14.608%\n",
      "Problem: X-n733-k159     Cost: 148786     Optimal Cost: 136187    \t Gap: 9.251%\n",
      "Problem: X-n749-k98      Cost: 85048      Optimal Cost: 77269     \t Gap: 10.067%\n",
      "Problem: X-n766-k71      Cost: 130052     Optimal Cost: 114417    \t Gap: 13.665%\n",
      "Problem: X-n783-k48      Cost: 83165      Optimal Cost: 72386     \t Gap: 14.891%\n",
      "Problem: X-n801-k40      Cost: 86024      Optimal Cost: 73311     \t Gap: 17.341%\n",
      "Problem: X-n819-k171     Cost: 174609     Optimal Cost: 158121    \t Gap: 10.427%\n",
      "Problem: X-n837-k142     Cost: 208252     Optimal Cost: 193737    \t Gap: 7.492%\n",
      "Problem: X-n856-k95      Cost: 98393      Optimal Cost: 88965     \t Gap: 10.597%\n",
      "Problem: X-n876-k59      Cost: 107229     Optimal Cost: 99299     \t Gap: 7.986%\n",
      "Problem: X-n895-k37      Cost: 64525      Optimal Cost: 53860     \t Gap: 19.801%\n",
      "Problem: X-n916-k207     Cost: 352732     Optimal Cost: 329179    \t Gap: 7.155%\n",
      "Problem: X-n936-k151     Cost: 163073     Optimal Cost: 132715    \t Gap: 22.875%\n",
      "Problem: X-n957-k87      Cost: 102964     Optimal Cost: 85465     \t Gap: 20.475%\n",
      "Problem: X-n979-k58      Cost: 129770     Optimal Cost: 118976    \t Gap: 9.072%\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "# Load\n",
    "with open(SAVEDIR+'rf-transformer.pkl', 'rb') as f:\n",
    "    results = pickle.load(f)\n",
    "\n",
    "# sort results by name\n",
    "results = sorted(results, key=lambda x: x[\"instance\"])\n",
    "\n",
    "for instance in results:\n",
    "    print(f'Problem: {instance[\"instance\"]:<15} Cost: {instance[\"cost\"]:<10} Optimal Cost: {instance[\"optimal_cost\"]:<10}\\t Gap: {instance[\"gap\"]:.3%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group A: 2.825% %\n",
      "Group B: 2.583% %\n",
      "Group E: 2.929% %\n",
      "Group F: 12.951% %\n",
      "Group M: 5.078% %\n",
      "Group P: 4.573% %\n",
      "Group X: 8.437% %\n"
     ]
    }
   ],
   "source": [
    "# Take the results. Measure gap depending on first letter of the instance name\n",
    "gaps_names = {}\n",
    "for instance in results:\n",
    "    name = instance['instance'][0]\n",
    "    # if name == 'X':\n",
    "    #     num_locs = instance['instance'].split('-')[1][1:]\n",
    "    #     if int(num_locs) < 252:\n",
    "    #         name = 'X<251'\n",
    "    #     elif int(num_locs) >= 252 and int(num_locs) <= 501:\n",
    "    #         name = 'X251-501'\n",
    "    #     else:\n",
    "    #         name = 'X501-1000'\n",
    "    if name not in gaps_names:\n",
    "        # if X, then we divide between < 251 and >= 251\n",
    "        gaps_names[name] = []\n",
    "    gaps_names[name].append(instance['gap'])\n",
    "        \n",
    "# Calculate the average gap for each group\n",
    "average_gaps = {name: sum(gaps) / len(gaps) for name, gaps in gaps_names.items()}\n",
    "for name, gap in average_gaps.items():\n",
    "    print(f'Group {name}: {(gap):.3%} %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Group A: 2.825% %\n",
      "Group B: 2.583% %\n",
      "Group E: 2.929% %\n",
      "Group F: 12.951% %\n",
      "Group M: 5.078% %\n",
      "Group P: 4.573% %\n",
      "Group X501-1000: 12.274% %\n",
      "Group X<251: 5.103% %\n",
      "Group X251-501: 8.072% %\n"
     ]
    }
   ],
   "source": [
    "# Take the results. Measure gap depending on first letter of the instance name\n",
    "gaps_names = {}\n",
    "for instance in results:\n",
    "    name = instance['instance'][0]\n",
    "    if name == 'X':\n",
    "        num_locs = instance['instance'].split('-')[1][1:]\n",
    "        if int(num_locs) < 252:\n",
    "            name = 'X<251'\n",
    "        elif int(num_locs) >= 252 and int(num_locs) <= 501:\n",
    "            name = 'X251-501'\n",
    "        else:\n",
    "            name = 'X501-1000'\n",
    "    if name not in gaps_names:\n",
    "        # if X, then we divide between < 251 and >= 251\n",
    "        gaps_names[name] = []\n",
    "    gaps_names[name].append(instance['gap'])\n",
    "        \n",
    "# Calculate the average gap for each group\n",
    "average_gaps = {name: sum(gaps) / len(gaps) for name, gaps in gaps_names.items()}\n",
    "for name, gap in average_gaps.items():\n",
    "    print(f'Group {name}: {(gap):.3%} %')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rl4co",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}