From 10a55ff6d02d733f7d24c4bb1f5618c876e643d6 Mon Sep 17 00:00:00 2001 From: Asugawara Date: Tue, 1 Dec 2020 20:55:53 +0900 Subject: [PATCH] add nfsp --- open_spiel/colabs/research_nfsp_tf_pt.ipynb | 3055 +++++++++++++++++ open_spiel/python/CMakeLists.txt | 1 + open_spiel/python/pytorch/nfsp.py | 343 ++ .../python/pytorch/nfsp_pytorch_test.py | 111 + 4 files changed, 3510 insertions(+) create mode 100644 open_spiel/colabs/research_nfsp_tf_pt.ipynb create mode 100644 open_spiel/python/pytorch/nfsp.py create mode 100644 open_spiel/python/pytorch/nfsp_pytorch_test.py diff --git a/open_spiel/colabs/research_nfsp_tf_pt.ipynb b/open_spiel/colabs/research_nfsp_tf_pt.ipynb new file mode 100644 index 0000000000..865d0953df --- /dev/null +++ b/open_spiel/colabs/research_nfsp_tf_pt.ipynb @@ -0,0 +1,3055 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /usr/local/lib/python3.8/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "non-resource variables are not supported in the long term\n" + ] + } + ], + "source": [ + "from absl import logging\n", + "import tensorflow.compat.v1 as tf\n", + "\n", + "from open_spiel.python import policy\n", + "from open_spiel.python import rl_environment\n", + "from open_spiel.python.algorithms import exploitability\n", + "from open_spiel.python.algorithms import nfsp\n", + "from open_spiel.python.pytorch import nfsp as nfsp_pt\n", + "\n", + "class NFSPPolicies(policy.Policy):\n", + " \"\"\"Joint policy to be evaluated.\"\"\"\n", + "\n", + " def __init__(self, env, nfsp_policies, mode):\n", + " game = env.game\n", + " player_ids = [0, 1]\n", + " super(NFSPPolicies, self).__init__(game, player_ids)\n", + " self._policies = nfsp_policies\n", + " self._mode = mode\n", + " self._obs = {\"info_state\": [None, None], \"legal_actions\": [None, None]}\n", + "\n", + " def action_probabilities(self, state, player_id=None):\n", + " cur_player = state.current_player()\n", + " legal_actions = state.legal_actions(cur_player)\n", + "\n", + " self._obs[\"current_player\"] = cur_player\n", + " self._obs[\"info_state\"][cur_player] = (\n", + " state.information_state_tensor(cur_player))\n", + " self._obs[\"legal_actions\"][cur_player] = legal_actions\n", + "\n", + " info_state = rl_environment.TimeStep(\n", + " observations=self._obs, rewards=None, discounts=None, step_type=None)\n", + "\n", + " with self._policies[cur_player].temp_mode_as(self._mode):\n", + " p = self._policies[cur_player].step(info_state, is_evaluation=True).probs\n", + " prob_dict = {action: p[action] for action in legal_actions}\n", + " return prob_dict\n", + "\n", + "\n", + "def tf_main(game,\n", + " env_config,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param):\n", + " env = rl_environment.Environment(game, **env_configs)\n", + " info_state_size = env.observation_spec()[\"info_state\"][0]\n", + " num_actions = env.action_spec()[\"num_actions\"]\n", + "\n", + " hidden_layers_sizes = [int(l) for l in hidden_layers_sizes]\n", + " kwargs = {\n", + " \"replay_buffer_capacity\": replay_buffer_capacity,\n", + " \"epsilon_decay_duration\": num_train_episodes,\n", + " \"epsilon_start\": 0.06,\n", + " \"epsilon_end\": 0.001,\n", + " }\n", + " expl_list = []\n", + " with tf.Session() as sess:\n", + " # pylint: disable=g-complex-comprehension\n", + " agents = [\n", + " nfsp.NFSP(sess, idx, info_state_size, num_actions, hidden_layers_sizes,\n", + " reservoir_buffer_capacity, anticipatory_param,\n", + " **kwargs) for idx in range(num_players)\n", + " ]\n", + " expl_policies_avg = NFSPPolicies(env, agents, nfsp.MODE.average_policy)\n", + "\n", + " sess.run(tf.global_variables_initializer())\n", + " for ep in range(num_train_episodes):\n", + " if (ep + 1) % eval_every == 0:\n", + " losses = [agent.loss for agent in agents]\n", + " print(\"Losses: %s\" %losses)\n", + " expl = exploitability.exploitability(env.game, expl_policies_avg)\n", + " expl_list.append(expl)\n", + " print(\"[%s] Exploitability AVG %s\" %(ep + 1, expl))\n", + " print(\"_____________________________________________\")\n", + "\n", + " time_step = env.reset()\n", + " while not time_step.last():\n", + " player_id = time_step.observations[\"current_player\"]\n", + " agent_output = agents[player_id].step(time_step)\n", + " action_list = [agent_output.action]\n", + " time_step = env.step(action_list)\n", + "\n", + " # Episode is over, step all agents with final info state.\n", + " for agent in agents:\n", + " agent.step(time_step)\n", + " return expl_list\n", + " \n", + "def pt_main(game,\n", + " env_config,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param):\n", + " env = rl_environment.Environment(game, **env_configs)\n", + " info_state_size = env.observation_spec()[\"info_state\"][0]\n", + " num_actions = env.action_spec()[\"num_actions\"]\n", + "\n", + " hidden_layers_sizes = [int(l) for l in hidden_layers_sizes]\n", + " kwargs = {\n", + " \"replay_buffer_capacity\": replay_buffer_capacity,\n", + " \"epsilon_decay_duration\": num_train_episodes,\n", + " \"epsilon_start\": 0.06,\n", + " \"epsilon_end\": 0.001,\n", + " }\n", + " expl_list = []\n", + " agents = [\n", + " nfsp_pt.NFSP(idx, info_state_size, num_actions, hidden_layers_sizes,\n", + " reservoir_buffer_capacity, anticipatory_param,\n", + " **kwargs) for idx in range(num_players)\n", + " ]\n", + " expl_policies_avg = NFSPPolicies(env, agents, nfsp_pt.MODE.average_policy) \n", + " for ep in range(num_train_episodes):\n", + " if (ep + 1) % eval_every == 0:\n", + " losses = [agent.loss.item() for agent in agents]\n", + " print(\"Losses: %s\" %losses)\n", + " expl = exploitability.exploitability(env.game, expl_policies_avg)\n", + " expl_list.append(expl)\n", + " print(\"[%s] Exploitability AVG %s\" %(ep + 1, expl))\n", + " print(\"_____________________________________________\") \n", + " time_step = env.reset()\n", + " while not time_step.last():\n", + " player_id = time_step.observations[\"current_player\"]\n", + " agent_output = agents[player_id].step(time_step)\n", + " action_list = [agent_output.action]\n", + " time_step = env.step(action_list) \n", + " # Episode is over, step all agents with final info state.\n", + " for agent in agents:\n", + " agent.step(time_step)\n", + " return expl_list" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "game = \"kuhn_poker\"\n", + "num_players = 2\n", + "env_configs = {\"players\": num_players}\n", + "num_train_episodes = int(3e6)\n", + "eval_every = 10000\n", + "hidden_layers_sizes = [128]\n", + "replay_buffer_capacity = int(2e5)\n", + "reservoir_buffer_capacity = int(2e6)\n", + "anticipatory_param = 0.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Losses: [(0.53785634, 0.9251124), (None, 0.9612581)]\n", + "[10000] Exploitability AVG 0.3806265065406539\n", + "_____________________________________________\n", + "Losses: [(0.36655694, 1.0659907), (0.3947149, 0.7292872)]\n", + "[20000] Exploitability AVG 0.25093042241453994\n", + "_____________________________________________\n", + "Losses: [(0.393549, 0.7483181), (0.24578282, 0.9105486)]\n", + "[30000] Exploitability AVG 0.2390693417084421\n", + "_____________________________________________\n", + "Losses: [(0.57156175, 0.89634085), (0.28940323, 1.1088376)]\n", + "[40000] Exploitability AVG 0.1969633608406444\n", + "_____________________________________________\n", + "Losses: [(0.5530685, 0.77027917), (0.3845034, 0.9829901)]\n", + "[50000] Exploitability AVG 0.1952863363167002\n", + "_____________________________________________\n", + "Losses: [(0.5764897, 1.05086), (0.26840705, 0.9912839)]\n", + "[60000] Exploitability AVG 0.1908509819947581\n", + "_____________________________________________\n", + "Losses: [(0.55077726, 0.9610203), (0.13408442, 0.8307791)]\n", + "[70000] Exploitability AVG 0.16578364111899666\n", + "_____________________________________________\n", + "Losses: [(0.5469289, 1.1176522), (0.2317084, 1.215792)]\n", + "[80000] Exploitability AVG 0.1516934982537071\n", + "_____________________________________________\n", + "Losses: [(0.5394142, 0.7201488), (0.232295, 0.6600412)]\n", + "[90000] Exploitability AVG 0.13771536690789724\n", + "_____________________________________________\n", + "Losses: [(0.5320931, 0.6304366), (0.2617296, 0.65807134)]\n", + "[100000] Exploitability AVG 0.13341837902810572\n", + "_____________________________________________\n", + "Losses: [(0.46678144, 0.8242874), (0.16980532, 1.0637206)]\n", + "[110000] Exploitability AVG 0.13644374987004593\n", + "_____________________________________________\n", + "Losses: [(0.43169168, 0.803023), (0.12921575, 1.0689111)]\n", + "[120000] Exploitability AVG 0.13798249975438104\n", + "_____________________________________________\n", + "Losses: [(0.47644025, 0.9729296), (0.17758624, 0.98064715)]\n", + "[130000] Exploitability AVG 0.14101250876721272\n", + "_____________________________________________\n", + "Losses: [(0.43854824, 0.5290735), (0.2415215, 1.0769482)]\n", + "[140000] Exploitability AVG 0.1441424008015637\n", + "_____________________________________________\n", + "Losses: [(0.301438, 0.74798745), (0.1424156, 0.6265968)]\n", + "[150000] Exploitability AVG 0.14635210990591047\n", + "_____________________________________________\n", + "Losses: [(0.50161743, 0.6071836), (0.21799245, 0.88656795)]\n", + "[160000] Exploitability AVG 0.14904867960404455\n", + "_____________________________________________\n", + "Losses: [(0.38752356, 0.6050333), (0.32411844, 0.6888297)]\n", + "[170000] Exploitability AVG 0.14844024467962766\n", + "_____________________________________________\n", + "Losses: [(0.3237788, 0.45750403), (0.19798557, 0.9790179)]\n", + "[180000] Exploitability AVG 0.1432817219201606\n", + "_____________________________________________\n", + "Losses: [(0.3172248, 0.3299994), (0.35026222, 0.7734553)]\n", + "[190000] Exploitability AVG 0.1374702476553379\n", + "_____________________________________________\n", + "Losses: [(0.34955397, 0.7731432), (0.23288381, 0.7341363)]\n", + "[200000] Exploitability AVG 0.13646102192037332\n", + "_____________________________________________\n", + "Losses: [(0.387213, 0.5182707), (0.31562436, 0.98022544)]\n", + "[210000] Exploitability AVG 0.13617992063928874\n", + "_____________________________________________\n", + "Losses: [(0.44026735, 0.30768), (0.28280202, 1.0117066)]\n", + "[220000] Exploitability AVG 0.13357250781103439\n", + "_____________________________________________\n", + "Losses: [(0.42528456, 0.58755875), (0.27889246, 0.7505833)]\n", + "[230000] Exploitability AVG 0.13009763608049202\n", + "_____________________________________________\n", + "Losses: [(0.3256826, 0.41950458), (0.29155385, 1.1112498)]\n", + "[240000] Exploitability AVG 0.12439814630878687\n", + "_____________________________________________\n", + "Losses: [(0.41040376, 0.5407282), (0.25253677, 1.1111819)]\n", + "[250000] Exploitability AVG 0.12094512628418666\n", + "_____________________________________________\n", + "Losses: [(0.42632166, 0.45957065), (0.32739842, 0.8982413)]\n", + "[260000] Exploitability AVG 0.11838708905427595\n", + "_____________________________________________\n", + "Losses: [(0.40546113, 0.31385136), (0.36196265, 0.64378595)]\n", + "[270000] Exploitability AVG 0.11397190181927891\n", + "_____________________________________________\n", + "Losses: [(0.3658614, 0.433307), (0.33337173, 0.7214786)]\n", + "[280000] Exploitability AVG 0.11403329146455082\n", + "_____________________________________________\n", + "Losses: [(0.3765139, 0.27072996), (0.30941635, 0.70414305)]\n", + "[290000] Exploitability AVG 0.10923850441120808\n", + "_____________________________________________\n", + "Losses: [(0.4114926, 0.42059997), (0.2661946, 0.67620766)]\n", + "[300000] Exploitability AVG 0.10810779550798735\n", + "_____________________________________________\n", + "Losses: [(0.4949669, 0.42543912), (0.3190732, 0.6458422)]\n", + "[310000] Exploitability AVG 0.10380301031311279\n", + "_____________________________________________\n", + "Losses: [(0.36090815, 0.5120726), (0.3166638, 0.6717366)]\n", + "[320000] Exploitability AVG 0.10169649320455076\n", + "_____________________________________________\n", + "Losses: [(0.4611076, 0.5772422), (0.38443658, 0.6642834)]\n", + "[330000] Exploitability AVG 0.09902836470968551\n", + "_____________________________________________\n", + "Losses: [(0.41778028, 0.3412873), (0.3854894, 0.86952704)]\n", + "[340000] Exploitability AVG 0.10004057069605216\n", + "_____________________________________________\n", + "Losses: [(0.4260533, 0.2604547), (0.28540164, 0.6263736)]\n", + "[350000] Exploitability AVG 0.10122644963548008\n", + "_____________________________________________\n", + "Losses: [(0.4273047, 0.3343315), (0.44963604, 0.47020024)]\n", + "[360000] Exploitability AVG 0.10525184712838637\n", + "_____________________________________________\n", + "Losses: [(0.39080882, 0.4416518), (0.27613676, 0.7313396)]\n", + "[370000] Exploitability AVG 0.10136377450585335\n", + "_____________________________________________\n", + "Losses: [(0.3096458, 0.31612632), (0.30239224, 0.54126966)]\n", + "[380000] Exploitability AVG 0.10103724952952689\n", + "_____________________________________________\n", + "Losses: [(0.3806877, 0.33446515), (0.36338568, 0.65576476)]\n", + "[390000] Exploitability AVG 0.10392499781552858\n", + "_____________________________________________\n", + "Losses: [(0.4079854, 0.57450235), (0.43979117, 0.61380184)]\n", + "[400000] Exploitability AVG 0.1023431129139718\n", + "_____________________________________________\n", + "Losses: [(0.40711147, 0.5788107), (0.32202923, 0.5339364)]\n", + "[410000] Exploitability AVG 0.10803959771600394\n", + "_____________________________________________\n", + "Losses: [(0.42193207, 0.2570755), (0.26632547, 0.40713254)]\n", + "[420000] Exploitability AVG 0.10621570549157452\n", + "_____________________________________________\n", + "Losses: [(0.35900956, 0.60050035), (0.3165278, 0.46993694)]\n", + "[430000] Exploitability AVG 0.10923746880004157\n", + "_____________________________________________\n", + "Losses: [(0.38367143, 0.38643956), (0.28973615, 0.5237889)]\n", + "[440000] Exploitability AVG 0.11087742833249944\n", + "_____________________________________________\n", + "Losses: [(0.48892343, 0.3828555), (0.35558242, 0.5800189)]\n", + "[450000] Exploitability AVG 0.10834048159432741\n", + "_____________________________________________\n", + "Losses: [(0.42650366, 0.45163384), (0.3151021, 0.5574181)]\n", + "[460000] Exploitability AVG 0.10583170394922509\n", + "_____________________________________________\n", + "Losses: [(0.3256737, 0.42145592), (0.34461606, 0.32864577)]\n", + "[470000] Exploitability AVG 0.10415989999256273\n", + "_____________________________________________\n", + "Losses: [(0.3198613, 0.41332233), (0.42090857, 0.47442433)]\n", + "[480000] Exploitability AVG 0.10504748207797285\n", + "_____________________________________________\n", + "Losses: [(0.32979816, 0.6369004), (0.44699854, 0.51994157)]\n", + "[490000] Exploitability AVG 0.10298133899582279\n", + "_____________________________________________\n", + "Losses: [(0.43519434, 0.35847187), (0.37962353, 0.5461072)]\n", + "[500000] Exploitability AVG 0.10144794777997385\n", + "_____________________________________________\n", + "Losses: [(0.394814, 0.5935895), (0.36149555, 0.50960845)]\n", + "[510000] Exploitability AVG 0.09834131354766457\n", + "_____________________________________________\n", + "Losses: [(0.3940258, 0.43741518), (0.39916283, 0.42474538)]\n", + "[520000] Exploitability AVG 0.09563283140000736\n", + "_____________________________________________\n", + "Losses: [(0.35646245, 0.34998295), (0.37988245, 0.31049204)]\n", + "[530000] Exploitability AVG 0.09244461235670032\n", + "_____________________________________________\n", + "Losses: [(0.38262355, 0.27072364), (0.37137628, 0.35718387)]\n", + "[540000] Exploitability AVG 0.09475821770630197\n", + "_____________________________________________\n", + "Losses: [(0.36840472, 0.5003272), (0.42205325, 0.34236807)]\n", + "[550000] Exploitability AVG 0.09229292608564099\n", + "_____________________________________________\n", + "Losses: [(0.4088319, 0.5113431), (0.3747962, 0.2870531)]\n", + "[560000] Exploitability AVG 0.09388690606227434\n", + "_____________________________________________\n", + "Losses: [(0.35466728, 0.49538395), (0.360672, 0.3025927)]\n", + "[570000] Exploitability AVG 0.09382583330219546\n", + "_____________________________________________\n", + "Losses: [(0.36435717, 0.35890487), (0.37422755, 0.39272133)]\n", + "[580000] Exploitability AVG 0.09058501698046251\n", + "_____________________________________________\n", + "Losses: [(0.40537262, 0.330426), (0.3503149, 0.8561445)]\n", + "[590000] Exploitability AVG 0.09011159745861785\n", + "_____________________________________________\n", + "Losses: [(0.42308077, 0.47072142), (0.41888517, 0.5398972)]\n", + "[600000] Exploitability AVG 0.09125654823545135\n", + "_____________________________________________\n", + "Losses: [(0.33426523, 0.49496457), (0.4176308, 0.6618978)]\n", + "[610000] Exploitability AVG 0.0876753745489307\n", + "_____________________________________________\n", + "Losses: [(0.39457476, 0.43405792), (0.38760054, 0.4215861)]\n", + "[620000] Exploitability AVG 0.08323237796951047\n", + "_____________________________________________\n", + "Losses: [(0.33993834, 0.2517962), (0.43690592, 0.28313082)]\n", + "[630000] Exploitability AVG 0.08539288231038411\n", + "_____________________________________________\n", + "Losses: [(0.34630015, 0.542068), (0.44489148, 0.3504649)]\n", + "[640000] Exploitability AVG 0.08617996800091454\n", + "_____________________________________________\n", + "Losses: [(0.40694013, 0.528067), (0.43522167, 0.43599227)]\n", + "[650000] Exploitability AVG 0.08598480283898413\n", + "_____________________________________________\n", + "Losses: [(0.32557255, 0.4392686), (0.40387726, 0.6786845)]\n", + "[660000] Exploitability AVG 0.09079874302152338\n", + "_____________________________________________\n", + "Losses: [(0.26883182, 0.51197803), (0.31877857, 0.4206481)]\n", + "[670000] Exploitability AVG 0.08742515412878249\n", + "_____________________________________________\n", + "Losses: [(0.3421637, 0.48392037), (0.4217058, 0.38747314)]\n", + "[680000] Exploitability AVG 0.0910077618182066\n", + "_____________________________________________\n", + "Losses: [(0.30845588, 0.34985754), (0.4298898, 0.31315628)]\n", + "[690000] Exploitability AVG 0.09061823818540501\n", + "_____________________________________________\n", + "Losses: [(0.3339134, 0.28532535), (0.41643834, 0.5775358)]\n", + "[700000] Exploitability AVG 0.09405131184145288\n", + "_____________________________________________\n", + "Losses: [(0.42452797, 0.39970428), (0.44133824, 0.58680904)]\n", + "[710000] Exploitability AVG 0.08834599153060632\n", + "_____________________________________________\n", + "Losses: [(0.43691695, 0.19820035), (0.45202005, 0.37203515)]\n", + "[720000] Exploitability AVG 0.091941853389547\n", + "_____________________________________________\n", + "Losses: [(0.36082175, 0.2868564), (0.4328217, 0.50956666)]\n", + "[730000] Exploitability AVG 0.0881980792656866\n", + "_____________________________________________\n", + "Losses: [(0.3220496, 0.4398561), (0.4897603, 0.46597838)]\n", + "[740000] Exploitability AVG 0.08776657289747197\n", + "_____________________________________________\n", + "Losses: [(0.40995064, 0.26855445), (0.4065367, 0.47443044)]\n", + "[750000] Exploitability AVG 0.0900312351993596\n", + "_____________________________________________\n", + "Losses: [(0.38390252, 0.5177217), (0.45436037, 0.6344828)]\n", + "[760000] Exploitability AVG 0.09179341292641752\n", + "_____________________________________________\n", + "Losses: [(0.32286388, 0.49822575), (0.44672647, 0.4767281)]\n", + "[770000] Exploitability AVG 0.09128744924573709\n", + "_____________________________________________\n", + "Losses: [(0.3593978, 0.529052), (0.39134938, 0.7177604)]\n", + "[780000] Exploitability AVG 0.09144012689076186\n", + "_____________________________________________\n", + "Losses: [(0.2805578, 0.32442468), (0.44368446, 0.6678139)]\n", + "[790000] Exploitability AVG 0.09029689746815744\n", + "_____________________________________________\n", + "Losses: [(0.35105583, 0.51136744), (0.41284293, 0.63977677)]\n", + "[800000] Exploitability AVG 0.08723216847369811\n", + "_____________________________________________\n", + "Losses: [(0.45607024, 0.4904994), (0.3347882, 0.6454823)]\n", + "[810000] Exploitability AVG 0.09069053667349367\n", + "_____________________________________________\n", + "Losses: [(0.3663447, 0.40416127), (0.36992317, 0.5616314)]\n", + "[820000] Exploitability AVG 0.0856810845853663\n", + "_____________________________________________\n", + "Losses: [(0.4000147, 0.4457639), (0.37032354, 0.5886855)]\n", + "[830000] Exploitability AVG 0.0888448333885746\n", + "_____________________________________________\n", + "Losses: [(0.4758286, 0.4686997), (0.3992513, 0.53458995)]\n", + "[840000] Exploitability AVG 0.08710027536043607\n", + "_____________________________________________\n", + "Losses: [(0.39632195, 0.465839), (0.4087019, 0.534943)]\n", + "[850000] Exploitability AVG 0.08882948153372575\n", + "_____________________________________________\n", + "Losses: [(0.41153845, 0.41162983), (0.3694026, 0.67539513)]\n", + "[860000] Exploitability AVG 0.08460102120668694\n", + "_____________________________________________\n", + "Losses: [(0.36325985, 0.6031791), (0.40446582, 0.5539188)]\n", + "[870000] Exploitability AVG 0.08914752846298232\n", + "_____________________________________________\n", + "Losses: [(0.43310434, 0.43803513), (0.36239192, 0.680726)]\n", + "[880000] Exploitability AVG 0.08339600323475704\n", + "_____________________________________________\n", + "Losses: [(0.38805214, 0.62767607), (0.39768362, 0.52786666)]\n", + "[890000] Exploitability AVG 0.0784939977014047\n", + "_____________________________________________\n", + "Losses: [(0.44102833, 0.4833071), (0.3844931, 0.6494594)]\n", + "[900000] Exploitability AVG 0.08041563564936269\n", + "_____________________________________________\n", + "Losses: [(0.41770506, 0.42907503), (0.3426093, 0.64371365)]\n", + "[910000] Exploitability AVG 0.08663348122719233\n", + "_____________________________________________\n", + "Losses: [(0.4698249, 0.5088979), (0.37519246, 0.8954325)]\n", + "[920000] Exploitability AVG 0.0882570630027745\n", + "_____________________________________________\n", + "Losses: [(0.4176745, 0.5089712), (0.3994038, 0.7921247)]\n", + "[930000] Exploitability AVG 0.09106935850947603\n", + "_____________________________________________\n", + "Losses: [(0.53730416, 0.58832085), (0.33749333, 0.65160495)]\n", + "[940000] Exploitability AVG 0.09102113047463317\n", + "_____________________________________________\n", + "Losses: [(0.40581885, 0.4593957), (0.35135883, 0.73974913)]\n", + "[950000] Exploitability AVG 0.0941877987959217\n", + "_____________________________________________\n", + "Losses: [(0.51756287, 0.5818273), (0.3701926, 0.6062665)]\n", + "[960000] Exploitability AVG 0.09074957818722046\n", + "_____________________________________________\n", + "Losses: [(0.4016741, 0.58563215), (0.41154104, 0.6222425)]\n", + "[970000] Exploitability AVG 0.09051007702080063\n", + "_____________________________________________\n", + "Losses: [(0.42308328, 0.5543698), (0.3405984, 0.84182686)]\n", + "[980000] Exploitability AVG 0.08676990763770578\n", + "_____________________________________________\n", + "Losses: [(0.49388278, 0.74858654), (0.4124651, 0.68941337)]\n", + "[990000] Exploitability AVG 0.08041205964379985\n", + "_____________________________________________\n", + "Losses: [(0.4595176, 0.5855596), (0.36452204, 0.5687382)]\n", + "[1000000] Exploitability AVG 0.08204261622396192\n", + "_____________________________________________\n", + "Losses: [(0.519078, 0.76983833), (0.4473907, 0.6541676)]\n", + "[1010000] Exploitability AVG 0.08564259044124173\n", + "_____________________________________________\n", + "Losses: [(0.41186434, 0.67432255), (0.43038964, 0.50249016)]\n", + "[1020000] Exploitability AVG 0.07862331933300507\n", + "_____________________________________________\n", + "Losses: [(0.4534778, 0.61701894), (0.31493905, 0.61674345)]\n", + "[1030000] Exploitability AVG 0.07806001708836008\n", + "_____________________________________________\n", + "Losses: [(0.5097435, 0.46884814), (0.41721335, 0.8037986)]\n", + "[1040000] Exploitability AVG 0.07944107747562015\n", + "_____________________________________________\n", + "Losses: [(0.48660356, 0.5661181), (0.40232286, 0.73411644)]\n", + "[1050000] Exploitability AVG 0.07721595169436243\n", + "_____________________________________________\n", + "Losses: [(0.4737168, 0.6015376), (0.46362174, 0.8082479)]\n", + "[1060000] Exploitability AVG 0.0769928429579155\n", + "_____________________________________________\n", + "Losses: [(0.46885955, 0.49490654), (0.39214587, 0.7677784)]\n", + "[1070000] Exploitability AVG 0.07631506069235242\n", + "_____________________________________________\n", + "Losses: [(0.45865953, 0.5850159), (0.37817693, 0.4762525)]\n", + "[1080000] Exploitability AVG 0.07482580942818043\n", + "_____________________________________________\n", + "Losses: [(0.4927109, 0.4080932), (0.39093417, 0.71216124)]\n", + "[1090000] Exploitability AVG 0.07766978736717581\n", + "_____________________________________________\n", + "Losses: [(0.45280257, 0.7883003), (0.3737149, 0.574195)]\n", + "[1100000] Exploitability AVG 0.07217197360168356\n", + "_____________________________________________\n", + "Losses: [(0.41287154, 0.5591948), (0.3722816, 0.5707054)]\n", + "[1110000] Exploitability AVG 0.0739192414478691\n", + "_____________________________________________\n", + "Losses: [(0.4144453, 0.6330303), (0.43275714, 0.6765839)]\n", + "[1120000] Exploitability AVG 0.06947669513073085\n", + "_____________________________________________\n", + "Losses: [(0.44436148, 0.6649499), (0.41045237, 0.6862391)]\n", + "[1130000] Exploitability AVG 0.07541265496083577\n", + "_____________________________________________\n", + "Losses: [(0.41057163, 0.44006693), (0.4189117, 0.84629536)]\n", + "[1140000] Exploitability AVG 0.07339804471533246\n", + "_____________________________________________\n", + "Losses: [(0.44088745, 0.55304503), (0.281748, 0.9063088)]\n", + "[1150000] Exploitability AVG 0.07184332737596394\n", + "_____________________________________________\n", + "Losses: [(0.4212719, 0.6008197), (0.3573202, 0.6705002)]\n", + "[1160000] Exploitability AVG 0.07126506261266255\n", + "_____________________________________________\n", + "Losses: [(0.3568734, 0.54455876), (0.40346837, 0.57008076)]\n", + "[1170000] Exploitability AVG 0.07009470934781589\n", + "_____________________________________________\n", + "Losses: [(0.4850189, 0.54863095), (0.41339225, 0.6310561)]\n", + "[1180000] Exploitability AVG 0.06714225217560521\n", + "_____________________________________________\n", + "Losses: [(0.47549695, 0.72816604), (0.36886775, 0.50604224)]\n", + "[1190000] Exploitability AVG 0.06668325267975705\n", + "_____________________________________________\n", + "Losses: [(0.45781267, 0.63168585), (0.3285314, 0.64426076)]\n", + "[1200000] Exploitability AVG 0.0659365316417648\n", + "_____________________________________________\n", + "Losses: [(0.5008249, 0.8449671), (0.37628335, 0.7850322)]\n", + "[1210000] Exploitability AVG 0.06648150585276588\n", + "_____________________________________________\n", + "Losses: [(0.48928463, 0.50049746), (0.33522755, 0.886652)]\n", + "[1220000] Exploitability AVG 0.06368644545091934\n", + "_____________________________________________\n", + "Losses: [(0.38201523, 0.4879728), (0.37080646, 0.48843157)]\n", + "[1230000] Exploitability AVG 0.06869178811599211\n", + "_____________________________________________\n", + "Losses: [(0.47955, 0.55848753), (0.3714931, 0.9279704)]\n", + "[1240000] Exploitability AVG 0.06195633740317863\n", + "_____________________________________________\n", + "Losses: [(0.4861322, 0.56263846), (0.433155, 0.6220056)]\n", + "[1250000] Exploitability AVG 0.06681131294861581\n", + "_____________________________________________\n", + "Losses: [(0.44647187, 0.6329416), (0.35019282, 0.6112708)]\n", + "[1260000] Exploitability AVG 0.06672105352638383\n", + "_____________________________________________\n", + "Losses: [(0.41749063, 0.789909), (0.39511335, 0.66746426)]\n", + "[1270000] Exploitability AVG 0.06403084592822672\n", + "_____________________________________________\n", + "Losses: [(0.47103474, 0.6849272), (0.36395818, 0.5995445)]\n", + "[1280000] Exploitability AVG 0.060835686925423654\n", + "_____________________________________________\n", + "Losses: [(0.45356104, 0.783978), (0.3700775, 0.6000366)]\n", + "[1290000] Exploitability AVG 0.06003043712866776\n", + "_____________________________________________\n", + "Losses: [(0.42446434, 0.7267907), (0.38503212, 0.71925336)]\n", + "[1300000] Exploitability AVG 0.058824696519878594\n", + "_____________________________________________\n", + "Losses: [(0.46800095, 0.5748359), (0.42482764, 0.72578067)]\n", + "[1310000] Exploitability AVG 0.059186150150376926\n", + "_____________________________________________\n", + "Losses: [(0.4082963, 0.62098587), (0.36195174, 0.6394584)]\n", + "[1320000] Exploitability AVG 0.05624888231580266\n", + "_____________________________________________\n", + "Losses: [(0.5021913, 0.54776615), (0.3284192, 0.5217803)]\n", + "[1330000] Exploitability AVG 0.058343054921288884\n", + "_____________________________________________\n", + "Losses: [(0.4307163, 0.8740953), (0.3860105, 0.6009859)]\n", + "[1340000] Exploitability AVG 0.057258419595942966\n", + "_____________________________________________\n", + "Losses: [(0.43308967, 0.6287751), (0.364323, 0.8880086)]\n", + "[1350000] Exploitability AVG 0.055347050763526334\n", + "_____________________________________________\n", + "Losses: [(0.42675477, 0.60907733), (0.33817303, 0.61862755)]\n", + "[1360000] Exploitability AVG 0.053996171807122795\n", + "_____________________________________________\n", + "Losses: [(0.5130402, 0.7870493), (0.32187206, 1.0502704)]\n", + "[1370000] Exploitability AVG 0.0559046419794165\n", + "_____________________________________________\n", + "Losses: [(0.39171708, 0.51615286), (0.34241295, 0.6908934)]\n", + "[1380000] Exploitability AVG 0.052203632353365476\n", + "_____________________________________________\n", + "Losses: [(0.43786377, 0.67642826), (0.31595203, 0.69594693)]\n", + "[1390000] Exploitability AVG 0.05153096895682391\n", + "_____________________________________________\n", + "Losses: [(0.35136995, 0.6251448), (0.42843956, 0.81424963)]\n", + "[1400000] Exploitability AVG 0.05372035209121906\n", + "_____________________________________________\n", + "Losses: [(0.40880853, 0.5171163), (0.39334106, 0.55050606)]\n", + "[1410000] Exploitability AVG 0.05082878725340234\n", + "_____________________________________________\n", + "Losses: [(0.4988798, 0.41859585), (0.34751773, 0.6486333)]\n", + "[1420000] Exploitability AVG 0.0534068216273742\n", + "_____________________________________________\n", + "Losses: [(0.4314149, 0.70552427), (0.42268878, 0.700639)]\n", + "[1430000] Exploitability AVG 0.05305497016857613\n", + "_____________________________________________\n", + "Losses: [(0.36766255, 0.73223627), (0.38321418, 0.58601844)]\n", + "[1440000] Exploitability AVG 0.05323394380411056\n", + "_____________________________________________\n", + "Losses: [(0.3452909, 0.79023486), (0.41286057, 0.5398059)]\n", + "[1450000] Exploitability AVG 0.052848428449435825\n", + "_____________________________________________\n", + "Losses: [(0.43301418, 0.5971317), (0.37514418, 0.5887003)]\n", + "[1460000] Exploitability AVG 0.05109147064226732\n", + "_____________________________________________\n", + "Losses: [(0.49610317, 0.7873826), (0.3663774, 0.8000857)]\n", + "[1470000] Exploitability AVG 0.05449714578171877\n", + "_____________________________________________\n", + "Losses: [(0.47720143, 0.48981217), (0.34160945, 0.94504327)]\n", + "[1480000] Exploitability AVG 0.053776967639592665\n", + "_____________________________________________\n", + "Losses: [(0.43578774, 0.46772003), (0.3675248, 0.6005969)]\n", + "[1490000] Exploitability AVG 0.05082950532222777\n", + "_____________________________________________\n", + "Losses: [(0.4170888, 0.5647226), (0.33423114, 0.53972244)]\n", + "[1500000] Exploitability AVG 0.05012266645152044\n", + "_____________________________________________\n", + "Losses: [(0.40586916, 0.58880734), (0.3595342, 0.40631717)]\n", + "[1510000] Exploitability AVG 0.046893708634786535\n", + "_____________________________________________\n", + "Losses: [(0.41815144, 0.6997858), (0.3520444, 0.7683254)]\n", + "[1520000] Exploitability AVG 0.0516005745835833\n", + "_____________________________________________\n", + "Losses: [(0.413977, 0.6843138), (0.3081952, 0.5857277)]\n", + "[1530000] Exploitability AVG 0.04967811329978106\n", + "_____________________________________________\n", + "Losses: [(0.4798899, 0.76630175), (0.38521272, 0.59402657)]\n", + "[1540000] Exploitability AVG 0.045311111641890334\n", + "_____________________________________________\n", + "Losses: [(0.48012662, 0.65406525), (0.3123728, 0.7236695)]\n", + "[1550000] Exploitability AVG 0.04571169229416333\n", + "_____________________________________________\n", + "Losses: [(0.37823898, 0.5858518), (0.3618425, 0.67495215)]\n", + "[1560000] Exploitability AVG 0.04596404449487956\n", + "_____________________________________________\n", + "Losses: [(0.46104524, 0.4383352), (0.32432604, 0.565593)]\n", + "[1570000] Exploitability AVG 0.0463707824651009\n", + "_____________________________________________\n", + "Losses: [(0.42663136, 0.41155636), (0.34510493, 0.4836244)]\n", + "[1580000] Exploitability AVG 0.04684576796732179\n", + "_____________________________________________\n", + "Losses: [(0.49639764, 0.56671435), (0.27595153, 0.60187733)]\n", + "[1590000] Exploitability AVG 0.044408465018632076\n", + "_____________________________________________\n", + "Losses: [(0.42510837, 0.4740823), (0.37934732, 0.60117894)]\n", + "[1600000] Exploitability AVG 0.045125104309523434\n", + "_____________________________________________\n", + "Losses: [(0.43871847, 0.5029764), (0.42387068, 0.70163983)]\n", + "[1610000] Exploitability AVG 0.04446031495127198\n", + "_____________________________________________\n", + "Losses: [(0.47028607, 0.61982685), (0.35155976, 0.7068691)]\n", + "[1620000] Exploitability AVG 0.04587300658014093\n", + "_____________________________________________\n", + "Losses: [(0.3585447, 0.6554501), (0.3251065, 0.52803266)]\n", + "[1630000] Exploitability AVG 0.048300014281447395\n", + "_____________________________________________\n", + "Losses: [(0.34247398, 0.64475167), (0.38979492, 0.95607865)]\n", + "[1640000] Exploitability AVG 0.042549473838091156\n", + "_____________________________________________\n", + "Losses: [(0.38869333, 0.4858371), (0.4564019, 0.50226915)]\n", + "[1650000] Exploitability AVG 0.04179842100102793\n", + "_____________________________________________\n", + "Losses: [(0.42086232, 0.7059016), (0.32230783, 0.60935646)]\n", + "[1660000] Exploitability AVG 0.042941556184754176\n", + "_____________________________________________\n", + "Losses: [(0.39194798, 0.78915286), (0.40711683, 0.5851205)]\n", + "[1670000] Exploitability AVG 0.04148112328807216\n", + "_____________________________________________\n", + "Losses: [(0.365149, 0.7423346), (0.35504484, 0.74086773)]\n", + "[1680000] Exploitability AVG 0.04205228785907761\n", + "_____________________________________________\n", + "Losses: [(0.3456332, 0.5866959), (0.28235933, 0.5748528)]\n", + "[1690000] Exploitability AVG 0.04237315083490009\n", + "_____________________________________________\n", + "Losses: [(0.42448598, 0.73854387), (0.26585948, 0.53774726)]\n", + "[1700000] Exploitability AVG 0.045948964624698774\n", + "_____________________________________________\n", + "Losses: [(0.33101755, 0.48256794), (0.29929924, 0.63849884)]\n", + "[1710000] Exploitability AVG 0.043802017472808014\n", + "_____________________________________________\n", + "Losses: [(0.44491085, 0.4412129), (0.3594916, 0.48978743)]\n", + "[1720000] Exploitability AVG 0.042861308532422454\n", + "_____________________________________________\n", + "Losses: [(0.457786, 0.52338827), (0.40888977, 0.5073445)]\n", + "[1730000] Exploitability AVG 0.041356282239048636\n", + "_____________________________________________\n", + "Losses: [(0.39555943, 0.7589325), (0.41768914, 0.64975107)]\n", + "[1740000] Exploitability AVG 0.04046556556923009\n", + "_____________________________________________\n", + "Losses: [(0.47985512, 0.620343), (0.3174764, 0.5914975)]\n", + "[1750000] Exploitability AVG 0.041343566341885934\n", + "_____________________________________________\n", + "Losses: [(0.43369815, 0.58176494), (0.38786978, 0.6104231)]\n", + "[1760000] Exploitability AVG 0.041906322746846386\n", + "_____________________________________________\n", + "Losses: [(0.41218752, 0.32087463), (0.28758323, 0.42452073)]\n", + "[1770000] Exploitability AVG 0.03830665181214732\n", + "_____________________________________________\n", + "Losses: [(0.40933692, 0.7664901), (0.33826464, 0.47491872)]\n", + "[1780000] Exploitability AVG 0.039177566450712964\n", + "_____________________________________________\n", + "Losses: [(0.40909323, 0.40401122), (0.32890928, 0.49418467)]\n", + "[1790000] Exploitability AVG 0.03922134585133824\n", + "_____________________________________________\n", + "Losses: [(0.42520168, 0.81759167), (0.38417363, 0.37097347)]\n", + "[1800000] Exploitability AVG 0.035762285233216035\n", + "_____________________________________________\n", + "Losses: [(0.46683937, 0.4862503), (0.35818103, 0.5575855)]\n", + "[1810000] Exploitability AVG 0.03791329144936029\n", + "_____________________________________________\n", + "Losses: [(0.4156337, 0.56349444), (0.3418883, 0.84977084)]\n", + "[1820000] Exploitability AVG 0.04010083316857141\n", + "_____________________________________________\n", + "Losses: [(0.48378587, 0.4667533), (0.3012209, 0.7336528)]\n", + "[1830000] Exploitability AVG 0.0377711491449218\n", + "_____________________________________________\n", + "Losses: [(0.3947456, 0.5656392), (0.36120027, 0.4233572)]\n", + "[1840000] Exploitability AVG 0.03925839021475638\n", + "_____________________________________________\n", + "Losses: [(0.42367142, 0.78298146), (0.3813373, 0.47584894)]\n", + "[1850000] Exploitability AVG 0.03772227416129739\n", + "_____________________________________________\n", + "Losses: [(0.4143026, 0.5004022), (0.33271122, 0.60987425)]\n", + "[1860000] Exploitability AVG 0.03657994944156415\n", + "_____________________________________________\n", + "Losses: [(0.39739615, 0.5869411), (0.3240341, 0.8148393)]\n", + "[1870000] Exploitability AVG 0.03672760384578044\n", + "_____________________________________________\n", + "Losses: [(0.52396923, 0.4500643), (0.40696603, 0.47574234)]\n", + "[1880000] Exploitability AVG 0.038535601316924295\n", + "_____________________________________________\n", + "Losses: [(0.4578817, 0.4258783), (0.37817496, 0.49975193)]\n", + "[1890000] Exploitability AVG 0.03931949228395368\n", + "_____________________________________________\n", + "Losses: [(0.4221358, 0.5085037), (0.32225513, 0.53039056)]\n", + "[1900000] Exploitability AVG 0.03627448017404572\n", + "_____________________________________________\n", + "Losses: [(0.4193602, 0.55534536), (0.3427239, 0.49866423)]\n", + "[1910000] Exploitability AVG 0.03702298422672179\n", + "_____________________________________________\n", + "Losses: [(0.37656474, 0.50582427), (0.29257947, 0.5263975)]\n", + "[1920000] Exploitability AVG 0.03602659092100108\n", + "_____________________________________________\n", + "Losses: [(0.37149176, 0.42294422), (0.4455578, 0.6109096)]\n", + "[1930000] Exploitability AVG 0.03619701896417182\n", + "_____________________________________________\n", + "Losses: [(0.42528695, 0.57070535), (0.36642998, 0.4506749)]\n", + "[1940000] Exploitability AVG 0.03530763845457155\n", + "_____________________________________________\n", + "Losses: [(0.4921395, 0.68806195), (0.33495793, 0.3977843)]\n", + "[1950000] Exploitability AVG 0.03412584203811472\n", + "_____________________________________________\n", + "Losses: [(0.41757697, 0.4617361), (0.3174243, 0.40962738)]\n", + "[1960000] Exploitability AVG 0.03607890274759737\n", + "_____________________________________________\n", + "Losses: [(0.4451823, 0.60289097), (0.34895226, 0.5212889)]\n", + "[1970000] Exploitability AVG 0.03193133111864316\n", + "_____________________________________________\n", + "Losses: [(0.49982548, 0.4113478), (0.37402153, 0.6020934)]\n", + "[1980000] Exploitability AVG 0.03468896850866751\n", + "_____________________________________________\n", + "Losses: [(0.38669774, 0.41071135), (0.2943524, 0.5957935)]\n", + "[1990000] Exploitability AVG 0.032152218052389825\n", + "_____________________________________________\n", + "Losses: [(0.4165921, 0.43808717), (0.34995088, 0.606176)]\n", + "[2000000] Exploitability AVG 0.030659672938502913\n", + "_____________________________________________\n", + "Losses: [(0.38585424, 0.35214698), (0.30226886, 0.5295311)]\n", + "[2010000] Exploitability AVG 0.029828780129660948\n", + "_____________________________________________\n", + "Losses: [(0.45095184, 0.48569152), (0.38080364, 0.4769916)]\n", + "[2020000] Exploitability AVG 0.02749860178148608\n", + "_____________________________________________\n", + "Losses: [(0.37607023, 0.645888), (0.3193574, 0.6802679)]\n", + "[2030000] Exploitability AVG 0.026810036151001332\n", + "_____________________________________________\n", + "Losses: [(0.3837772, 0.87497246), (0.38007155, 0.53103006)]\n", + "[2040000] Exploitability AVG 0.026867827541341288\n", + "_____________________________________________\n", + "Losses: [(0.5031503, 0.54775786), (0.3174397, 0.4067785)]\n", + "[2050000] Exploitability AVG 0.029507655913413033\n", + "_____________________________________________\n", + "Losses: [(0.39763346, 0.5429309), (0.38321906, 0.48586804)]\n", + "[2060000] Exploitability AVG 0.03021038629563494\n", + "_____________________________________________\n", + "Losses: [(0.35012954, 0.43252212), (0.39944032, 0.5198289)]\n", + "[2070000] Exploitability AVG 0.026712738691466564\n", + "_____________________________________________\n", + "Losses: [(0.3419823, 0.5396731), (0.43592724, 0.8773295)]\n", + "[2080000] Exploitability AVG 0.02501522728590347\n", + "_____________________________________________\n", + "Losses: [(0.42998898, 0.62838113), (0.3607468, 0.54565656)]\n", + "[2090000] Exploitability AVG 0.023993186254462645\n", + "_____________________________________________\n", + "Losses: [(0.4937744, 0.54657567), (0.28140676, 0.59794414)]\n", + "[2100000] Exploitability AVG 0.02425871255442577\n", + "_____________________________________________\n", + "Losses: [(0.4560567, 0.47020036), (0.33346045, 0.50051165)]\n", + "[2110000] Exploitability AVG 0.02718084409547153\n", + "_____________________________________________\n", + "Losses: [(0.35283047, 0.5554212), (0.26530296, 0.7174857)]\n", + "[2120000] Exploitability AVG 0.026133773265009264\n", + "_____________________________________________\n", + "Losses: [(0.4071419, 0.63623357), (0.33543956, 0.71079785)]\n", + "[2130000] Exploitability AVG 0.02570410258360384\n", + "_____________________________________________\n", + "Losses: [(0.42037562, 0.58466065), (0.39294472, 0.6559927)]\n", + "[2140000] Exploitability AVG 0.025383958497101955\n", + "_____________________________________________\n", + "Losses: [(0.38667828, 0.3897876), (0.4057403, 0.49722636)]\n", + "[2150000] Exploitability AVG 0.025208251606318066\n", + "_____________________________________________\n", + "Losses: [(0.43192378, 0.56289566), (0.35186633, 0.5607218)]\n", + "[2160000] Exploitability AVG 0.025779626746067985\n", + "_____________________________________________\n", + "Losses: [(0.4534146, 0.42264727), (0.37919503, 0.615955)]\n", + "[2170000] Exploitability AVG 0.024984274257347894\n", + "_____________________________________________\n", + "Losses: [(0.44034547, 0.37940258), (0.43951342, 0.49089834)]\n", + "[2180000] Exploitability AVG 0.023462354291754473\n", + "_____________________________________________\n", + "Losses: [(0.37985033, 0.47816586), (0.35868245, 0.45911416)]\n", + "[2190000] Exploitability AVG 0.025926467353787758\n", + "_____________________________________________\n", + "Losses: [(0.444169, 0.5961399), (0.3668325, 0.29173326)]\n", + "[2200000] Exploitability AVG 0.02461458153612084\n", + "_____________________________________________\n", + "Losses: [(0.39104006, 0.41651952), (0.32722044, 0.46530294)]\n", + "[2210000] Exploitability AVG 0.0244719229243385\n", + "_____________________________________________\n", + "Losses: [(0.40847737, 0.7047869), (0.3430224, 0.55030024)]\n", + "[2220000] Exploitability AVG 0.023827386820531787\n", + "_____________________________________________\n", + "Losses: [(0.43505037, 0.38895172), (0.34677938, 0.39898372)]\n", + "[2230000] Exploitability AVG 0.021715131833923734\n", + "_____________________________________________\n", + "Losses: [(0.50462234, 0.39230603), (0.34698427, 0.5375998)]\n", + "[2240000] Exploitability AVG 0.023650549578733637\n", + "_____________________________________________\n", + "Losses: [(0.36260206, 0.7000131), (0.34915394, 0.6204957)]\n", + "[2250000] Exploitability AVG 0.02362541373298263\n", + "_____________________________________________\n", + "Losses: [(0.31541502, 0.3721706), (0.30064937, 0.50012195)]\n", + "[2260000] Exploitability AVG 0.021479627873028678\n", + "_____________________________________________\n", + "Losses: [(0.3930034, 0.6512054), (0.3014471, 0.4627043)]\n", + "[2270000] Exploitability AVG 0.020422519088821117\n", + "_____________________________________________\n", + "Losses: [(0.43734628, 0.5186713), (0.37854224, 0.6141143)]\n", + "[2280000] Exploitability AVG 0.021943126759245257\n", + "_____________________________________________\n", + "Losses: [(0.4224162, 0.43876258), (0.32896662, 0.5478755)]\n", + "[2290000] Exploitability AVG 0.021461419739899235\n", + "_____________________________________________\n", + "Losses: [(0.4398613, 0.62656033), (0.3029998, 0.49637628)]\n", + "[2300000] Exploitability AVG 0.021432345928219937\n", + "_____________________________________________\n", + "Losses: [(0.43400115, 0.52699685), (0.35363263, 0.4040384)]\n", + "[2310000] Exploitability AVG 0.020503823155564677\n", + "_____________________________________________\n", + "Losses: [(0.4493752, 0.44873637), (0.34471804, 0.7119631)]\n", + "[2320000] Exploitability AVG 0.02282087490465534\n", + "_____________________________________________\n", + "Losses: [(0.2971115, 0.45127457), (0.35623252, 0.5549748)]\n", + "[2330000] Exploitability AVG 0.020094964423322675\n", + "_____________________________________________\n", + "Losses: [(0.3625483, 0.6027725), (0.3960712, 0.621154)]\n", + "[2340000] Exploitability AVG 0.019418982391820966\n", + "_____________________________________________\n", + "Losses: [(0.39599693, 0.7384269), (0.288099, 0.46412212)]\n", + "[2350000] Exploitability AVG 0.018934022496258557\n", + "_____________________________________________\n", + "Losses: [(0.36512956, 0.5814568), (0.27080905, 0.44266248)]\n", + "[2360000] Exploitability AVG 0.02128725563879527\n", + "_____________________________________________\n", + "Losses: [(0.4307524, 0.3962652), (0.35692203, 0.66424286)]\n", + "[2370000] Exploitability AVG 0.021917888095929094\n", + "_____________________________________________\n", + "Losses: [(0.4712869, 0.8447703), (0.30220306, 0.36533138)]\n", + "[2380000] Exploitability AVG 0.02028653221452617\n", + "_____________________________________________\n", + "Losses: [(0.4689538, 0.4786887), (0.35364628, 0.39933825)]\n", + "[2390000] Exploitability AVG 0.016303058626044142\n", + "_____________________________________________\n", + "Losses: [(0.38246992, 0.49547318), (0.3186066, 0.56552804)]\n", + "[2400000] Exploitability AVG 0.018320424887928705\n", + "_____________________________________________\n", + "Losses: [(0.4404869, 0.744156), (0.36639327, 0.64630365)]\n", + "[2410000] Exploitability AVG 0.01915758843377316\n", + "_____________________________________________\n", + "Losses: [(0.42716026, 0.50715494), (0.31562263, 0.45685565)]\n", + "[2420000] Exploitability AVG 0.017412583722899666\n", + "_____________________________________________\n", + "Losses: [(0.45215356, 0.6958087), (0.3088422, 0.45675677)]\n", + "[2430000] Exploitability AVG 0.020750307603143642\n", + "_____________________________________________\n", + "Losses: [(0.4379656, 0.37722975), (0.31273022, 0.6630598)]\n", + "[2440000] Exploitability AVG 0.018948478052878526\n", + "_____________________________________________\n", + "Losses: [(0.36716747, 0.6394386), (0.2987432, 0.54639834)]\n", + "[2450000] Exploitability AVG 0.020072371155766183\n", + "_____________________________________________\n", + "Losses: [(0.34182268, 0.56389123), (0.38061082, 0.47611558)]\n", + "[2460000] Exploitability AVG 0.01622033783633947\n", + "_____________________________________________\n", + "Losses: [(0.45655447, 0.6853083), (0.3305947, 0.65729046)]\n", + "[2470000] Exploitability AVG 0.017373402331382154\n", + "_____________________________________________\n", + "Losses: [(0.5503781, 0.607173), (0.35136533, 0.59352386)]\n", + "[2480000] Exploitability AVG 0.016252908387878462\n", + "_____________________________________________\n", + "Losses: [(0.47121713, 0.4388122), (0.30423117, 0.33464283)]\n", + "[2490000] Exploitability AVG 0.01524356132103849\n", + "_____________________________________________\n", + "Losses: [(0.464693, 0.4995058), (0.3393593, 0.66531265)]\n", + "[2500000] Exploitability AVG 0.01667942963435906\n", + "_____________________________________________\n", + "Losses: [(0.44602466, 0.5245248), (0.29780456, 0.66998994)]\n", + "[2510000] Exploitability AVG 0.018168945372750683\n", + "_____________________________________________\n", + "Losses: [(0.46063453, 0.48700446), (0.3708353, 0.49021626)]\n", + "[2520000] Exploitability AVG 0.01717999895103886\n", + "_____________________________________________\n", + "Losses: [(0.44870824, 0.56247914), (0.34475052, 0.47730786)]\n", + "[2530000] Exploitability AVG 0.014169907203878979\n", + "_____________________________________________\n", + "Losses: [(0.40798396, 0.59644943), (0.300268, 0.82246596)]\n", + "[2540000] Exploitability AVG 0.016110433558226556\n", + "_____________________________________________\n", + "Losses: [(0.40910918, 0.81199443), (0.35904902, 0.5672656)]\n", + "[2550000] Exploitability AVG 0.016911989347595968\n", + "_____________________________________________\n", + "Losses: [(0.40757793, 0.52735543), (0.3948415, 0.64401335)]\n", + "[2560000] Exploitability AVG 0.016918611139976197\n", + "_____________________________________________\n", + "Losses: [(0.42761454, 0.7518471), (0.36559847, 0.4864186)]\n", + "[2570000] Exploitability AVG 0.015861555493228152\n", + "_____________________________________________\n", + "Losses: [(0.42384738, 0.5069518), (0.33849835, 0.6077436)]\n", + "[2580000] Exploitability AVG 0.015886593668532623\n", + "_____________________________________________\n", + "Losses: [(0.45255536, 0.656397), (0.28774497, 0.79069686)]\n", + "[2590000] Exploitability AVG 0.019979591242851674\n", + "_____________________________________________\n", + "Losses: [(0.4211589, 0.74649644), (0.3714892, 0.68423676)]\n", + "[2600000] Exploitability AVG 0.016244876500421668\n", + "_____________________________________________\n", + "Losses: [(0.39316362, 0.68157417), (0.34940225, 0.34818918)]\n", + "[2610000] Exploitability AVG 0.01661257430438337\n", + "_____________________________________________\n", + "Losses: [(0.44703525, 0.57999), (0.31449428, 0.7077742)]\n", + "[2620000] Exploitability AVG 0.018034229228481857\n", + "_____________________________________________\n", + "Losses: [(0.42034236, 0.6279874), (0.36980236, 0.38621294)]\n", + "[2630000] Exploitability AVG 0.016888369906079337\n", + "_____________________________________________\n", + "Losses: [(0.3704744, 0.6143775), (0.31929296, 0.6217122)]\n", + "[2640000] Exploitability AVG 0.019784283509299783\n", + "_____________________________________________\n", + "Losses: [(0.39182502, 0.51257694), (0.4168296, 0.7646626)]\n", + "[2650000] Exploitability AVG 0.017430895423644593\n", + "_____________________________________________\n", + "Losses: [(0.41393298, 0.5444832), (0.3123076, 0.62097526)]\n", + "[2660000] Exploitability AVG 0.01837931725406791\n", + "_____________________________________________\n", + "Losses: [(0.55504227, 0.6852316), (0.31834304, 0.6725337)]\n", + "[2670000] Exploitability AVG 0.020167696108900457\n", + "_____________________________________________\n", + "Losses: [(0.4527389, 0.57109547), (0.33951366, 0.60364854)]\n", + "[2680000] Exploitability AVG 0.018854506971541146\n", + "_____________________________________________\n", + "Losses: [(0.4499932, 0.34525734), (0.37019715, 0.7622739)]\n", + "[2690000] Exploitability AVG 0.018470328625503712\n", + "_____________________________________________\n", + "Losses: [(0.4627331, 0.6056986), (0.31651556, 0.563763)]\n", + "[2700000] Exploitability AVG 0.01825374339010452\n", + "_____________________________________________\n", + "Losses: [(0.43327004, 0.4200788), (0.4006771, 0.52632415)]\n", + "[2710000] Exploitability AVG 0.018367022921516846\n", + "_____________________________________________\n", + "Losses: [(0.37335753, 0.6257259), (0.36139315, 0.5939001)]\n", + "[2720000] Exploitability AVG 0.019696022561432547\n", + "_____________________________________________\n", + "Losses: [(0.42385167, 0.5374807), (0.30151203, 0.51207596)]\n", + "[2730000] Exploitability AVG 0.021787655114860333\n", + "_____________________________________________\n", + "Losses: [(0.40455925, 0.5134685), (0.40346825, 0.5165838)]\n", + "[2740000] Exploitability AVG 0.019540489850477488\n", + "_____________________________________________\n", + "Losses: [(0.36694264, 0.66247094), (0.35681474, 0.3897757)]\n", + "[2750000] Exploitability AVG 0.01965555861963958\n", + "_____________________________________________\n", + "Losses: [(0.46748698, 0.7471348), (0.36174726, 0.56847614)]\n", + "[2760000] Exploitability AVG 0.01910709823166229\n", + "_____________________________________________\n", + "Losses: [(0.51304895, 0.33871382), (0.37971178, 0.5909578)]\n", + "[2770000] Exploitability AVG 0.021611410783969\n", + "_____________________________________________\n", + "Losses: [(0.488518, 0.522284), (0.30418184, 0.6986019)]\n", + "[2780000] Exploitability AVG 0.018934332443253765\n", + "_____________________________________________\n", + "Losses: [(0.43590057, 0.76400435), (0.31612486, 0.69220066)]\n", + "[2790000] Exploitability AVG 0.020873204793502076\n", + "_____________________________________________\n", + "Losses: [(0.44053298, 0.7072829), (0.228215, 0.50917506)]\n", + "[2800000] Exploitability AVG 0.017905055792117153\n", + "_____________________________________________\n", + "Losses: [(0.46812955, 0.5855954), (0.37111604, 0.5579703)]\n", + "[2810000] Exploitability AVG 0.02131898649672026\n", + "_____________________________________________\n", + "Losses: [(0.4276143, 0.43673497), (0.34636343, 0.48543942)]\n", + "[2820000] Exploitability AVG 0.020336371463632846\n", + "_____________________________________________\n", + "Losses: [(0.4609828, 0.6742366), (0.33348134, 0.39611524)]\n", + "[2830000] Exploitability AVG 0.020498945056803664\n", + "_____________________________________________\n", + "Losses: [(0.39039713, 0.7395321), (0.41318536, 0.4225493)]\n", + "[2840000] Exploitability AVG 0.018874717205044023\n", + "_____________________________________________\n", + "Losses: [(0.5095199, 0.44726318), (0.2723924, 0.5461652)]\n", + "[2850000] Exploitability AVG 0.019394350497066642\n", + "_____________________________________________\n", + "Losses: [(0.49325442, 0.7607732), (0.3477291, 0.9342608)]\n", + "[2860000] Exploitability AVG 0.018721171379744606\n", + "_____________________________________________\n", + "Losses: [(0.4537952, 0.65215737), (0.3413659, 0.62802666)]\n", + "[2870000] Exploitability AVG 0.014928025292856556\n", + "_____________________________________________\n", + "Losses: [(0.52189296, 0.79933834), (0.2513857, 0.6386173)]\n", + "[2880000] Exploitability AVG 0.01619391880665169\n", + "_____________________________________________\n", + "Losses: [(0.40829802, 0.61610216), (0.3708099, 0.5832578)]\n", + "[2890000] Exploitability AVG 0.02007720814287578\n", + "_____________________________________________\n", + "Losses: [(0.43133312, 0.41750568), (0.32444715, 0.68169194)]\n", + "[2900000] Exploitability AVG 0.01824893872654071\n", + "_____________________________________________\n", + "Losses: [(0.40551984, 0.46390215), (0.2969731, 0.6886542)]\n", + "[2910000] Exploitability AVG 0.02001883230695936\n", + "_____________________________________________\n", + "Losses: [(0.40584671, 0.5593766), (0.36044037, 0.52924776)]\n", + "[2920000] Exploitability AVG 0.01773258950644646\n", + "_____________________________________________\n", + "Losses: [(0.43411812, 0.6366219), (0.3400984, 0.617037)]\n", + "[2930000] Exploitability AVG 0.01817986292463314\n", + "_____________________________________________\n", + "Losses: [(0.4543123, 0.7383058), (0.36597294, 0.5555818)]\n", + "[2940000] Exploitability AVG 0.019548768034712283\n", + "_____________________________________________\n", + "Losses: [(0.50876534, 0.8629745), (0.37576485, 0.38417238)]\n", + "[2950000] Exploitability AVG 0.016988870253343002\n", + "_____________________________________________\n", + "Losses: [(0.46998835, 0.46722782), (0.36729643, 0.5773537)]\n", + "[2960000] Exploitability AVG 0.02043270049790577\n", + "_____________________________________________\n", + "Losses: [(0.40146762, 0.87588155), (0.29805446, 0.35675085)]\n", + "[2970000] Exploitability AVG 0.01534469283520995\n", + "_____________________________________________\n", + "Losses: [(0.42576748, 0.5401617), (0.35060087, 0.41815045)]\n", + "[2980000] Exploitability AVG 0.017446239863274676\n", + "_____________________________________________\n", + "Losses: [(0.4338294, 0.5503223), (0.34887776, 0.5909028)]\n", + "[2990000] Exploitability AVG 0.01712435226787648\n", + "_____________________________________________\n", + "Losses: [(0.41873193, 0.7645945), (0.33788505, 0.52467763)]\n", + "[3000000] Exploitability AVG 0.017898903855360915\n", + "_____________________________________________\n" + ] + }, + { + "data": { + "text/plain": [ + "[0.3806265065406539,\n", + " 0.25093042241453994,\n", + " 0.2390693417084421,\n", + " 0.1969633608406444,\n", + " 0.1952863363167002,\n", + " 0.1908509819947581,\n", + " 0.16578364111899666,\n", + " 0.1516934982537071,\n", + " 0.13771536690789724,\n", + " 0.13341837902810572,\n", + " 0.13644374987004593,\n", + " 0.13798249975438104,\n", + " 0.14101250876721272,\n", + " 0.1441424008015637,\n", + " 0.14635210990591047,\n", + " 0.14904867960404455,\n", + " 0.14844024467962766,\n", + " 0.1432817219201606,\n", + " 0.1374702476553379,\n", + " 0.13646102192037332,\n", + " 0.13617992063928874,\n", + " 0.13357250781103439,\n", + " 0.13009763608049202,\n", + " 0.12439814630878687,\n", + " 0.12094512628418666,\n", + " 0.11838708905427595,\n", + " 0.11397190181927891,\n", + " 0.11403329146455082,\n", + " 0.10923850441120808,\n", + " 0.10810779550798735,\n", + " 0.10380301031311279,\n", + " 0.10169649320455076,\n", + " 0.09902836470968551,\n", + " 0.10004057069605216,\n", + " 0.10122644963548008,\n", + " 0.10525184712838637,\n", + " 0.10136377450585335,\n", + " 0.10103724952952689,\n", + " 0.10392499781552858,\n", + " 0.1023431129139718,\n", + " 0.10803959771600394,\n", + " 0.10621570549157452,\n", + " 0.10923746880004157,\n", + " 0.11087742833249944,\n", + " 0.10834048159432741,\n", + " 0.10583170394922509,\n", + " 0.10415989999256273,\n", + " 0.10504748207797285,\n", + " 0.10298133899582279,\n", + " 0.10144794777997385,\n", + " 0.09834131354766457,\n", + " 0.09563283140000736,\n", + " 0.09244461235670032,\n", + " 0.09475821770630197,\n", + " 0.09229292608564099,\n", + " 0.09388690606227434,\n", + " 0.09382583330219546,\n", + " 0.09058501698046251,\n", + " 0.09011159745861785,\n", + " 0.09125654823545135,\n", + " 0.0876753745489307,\n", + " 0.08323237796951047,\n", + " 0.08539288231038411,\n", + " 0.08617996800091454,\n", + " 0.08598480283898413,\n", + " 0.09079874302152338,\n", + " 0.08742515412878249,\n", + " 0.0910077618182066,\n", + " 0.09061823818540501,\n", + " 0.09405131184145288,\n", + " 0.08834599153060632,\n", + " 0.091941853389547,\n", + " 0.0881980792656866,\n", + " 0.08776657289747197,\n", + " 0.0900312351993596,\n", + " 0.09179341292641752,\n", + " 0.09128744924573709,\n", + " 0.09144012689076186,\n", + " 0.09029689746815744,\n", + " 0.08723216847369811,\n", + " 0.09069053667349367,\n", + " 0.0856810845853663,\n", + " 0.0888448333885746,\n", + " 0.08710027536043607,\n", + " 0.08882948153372575,\n", + " 0.08460102120668694,\n", + " 0.08914752846298232,\n", + " 0.08339600323475704,\n", + " 0.0784939977014047,\n", + " 0.08041563564936269,\n", + " 0.08663348122719233,\n", + " 0.0882570630027745,\n", + " 0.09106935850947603,\n", + " 0.09102113047463317,\n", + " 0.0941877987959217,\n", + " 0.09074957818722046,\n", + " 0.09051007702080063,\n", + " 0.08676990763770578,\n", + " 0.08041205964379985,\n", + " 0.08204261622396192,\n", + " 0.08564259044124173,\n", + " 0.07862331933300507,\n", + " 0.07806001708836008,\n", + " 0.07944107747562015,\n", + " 0.07721595169436243,\n", + " 0.0769928429579155,\n", + " 0.07631506069235242,\n", + " 0.07482580942818043,\n", + " 0.07766978736717581,\n", + " 0.07217197360168356,\n", + " 0.0739192414478691,\n", + " 0.06947669513073085,\n", + " 0.07541265496083577,\n", + " 0.07339804471533246,\n", + " 0.07184332737596394,\n", + " 0.07126506261266255,\n", + " 0.07009470934781589,\n", + " 0.06714225217560521,\n", + " 0.06668325267975705,\n", + " 0.0659365316417648,\n", + " 0.06648150585276588,\n", + " 0.06368644545091934,\n", + " 0.06869178811599211,\n", + " 0.06195633740317863,\n", + " 0.06681131294861581,\n", + " 0.06672105352638383,\n", + " 0.06403084592822672,\n", + " 0.060835686925423654,\n", + " 0.06003043712866776,\n", + " 0.058824696519878594,\n", + " 0.059186150150376926,\n", + " 0.05624888231580266,\n", + " 0.058343054921288884,\n", + " 0.057258419595942966,\n", + " 0.055347050763526334,\n", + " 0.053996171807122795,\n", + " 0.0559046419794165,\n", + " 0.052203632353365476,\n", + " 0.05153096895682391,\n", + " 0.05372035209121906,\n", + " 0.05082878725340234,\n", + " 0.0534068216273742,\n", + " 0.05305497016857613,\n", + " 0.05323394380411056,\n", + " 0.052848428449435825,\n", + " 0.05109147064226732,\n", + " 0.05449714578171877,\n", + " 0.053776967639592665,\n", + " 0.05082950532222777,\n", + " 0.05012266645152044,\n", + " 0.046893708634786535,\n", + " 0.0516005745835833,\n", + " 0.04967811329978106,\n", + " 0.045311111641890334,\n", + " 0.04571169229416333,\n", + " 0.04596404449487956,\n", + " 0.0463707824651009,\n", + " 0.04684576796732179,\n", + " 0.044408465018632076,\n", + " 0.045125104309523434,\n", + " 0.04446031495127198,\n", + " 0.04587300658014093,\n", + " 0.048300014281447395,\n", + " 0.042549473838091156,\n", + " 0.04179842100102793,\n", + " 0.042941556184754176,\n", + " 0.04148112328807216,\n", + " 0.04205228785907761,\n", + " 0.04237315083490009,\n", + " 0.045948964624698774,\n", + " 0.043802017472808014,\n", + " 0.042861308532422454,\n", + " 0.041356282239048636,\n", + " 0.04046556556923009,\n", + " 0.041343566341885934,\n", + " 0.041906322746846386,\n", + " 0.03830665181214732,\n", + " 0.039177566450712964,\n", + " 0.03922134585133824,\n", + " 0.035762285233216035,\n", + " 0.03791329144936029,\n", + " 0.04010083316857141,\n", + " 0.0377711491449218,\n", + " 0.03925839021475638,\n", + " 0.03772227416129739,\n", + " 0.03657994944156415,\n", + " 0.03672760384578044,\n", + " 0.038535601316924295,\n", + " 0.03931949228395368,\n", + " 0.03627448017404572,\n", + " 0.03702298422672179,\n", + " 0.03602659092100108,\n", + " 0.03619701896417182,\n", + " 0.03530763845457155,\n", + " 0.03412584203811472,\n", + " 0.03607890274759737,\n", + " 0.03193133111864316,\n", + " 0.03468896850866751,\n", + " 0.032152218052389825,\n", + " 0.030659672938502913,\n", + " 0.029828780129660948,\n", + " 0.02749860178148608,\n", + " 0.026810036151001332,\n", + " 0.026867827541341288,\n", + " 0.029507655913413033,\n", + " 0.03021038629563494,\n", + " 0.026712738691466564,\n", + " 0.02501522728590347,\n", + " 0.023993186254462645,\n", + " 0.02425871255442577,\n", + " 0.02718084409547153,\n", + " 0.026133773265009264,\n", + " 0.02570410258360384,\n", + " 0.025383958497101955,\n", + " 0.025208251606318066,\n", + " 0.025779626746067985,\n", + " 0.024984274257347894,\n", + " 0.023462354291754473,\n", + " 0.025926467353787758,\n", + " 0.02461458153612084,\n", + " 0.0244719229243385,\n", + " 0.023827386820531787,\n", + " 0.021715131833923734,\n", + " 0.023650549578733637,\n", + " 0.02362541373298263,\n", + " 0.021479627873028678,\n", + " 0.020422519088821117,\n", + " 0.021943126759245257,\n", + " 0.021461419739899235,\n", + " 0.021432345928219937,\n", + " 0.020503823155564677,\n", + " 0.02282087490465534,\n", + " 0.020094964423322675,\n", + " 0.019418982391820966,\n", + " 0.018934022496258557,\n", + " 0.02128725563879527,\n", + " 0.021917888095929094,\n", + " 0.02028653221452617,\n", + " 0.016303058626044142,\n", + " 0.018320424887928705,\n", + " 0.01915758843377316,\n", + " 0.017412583722899666,\n", + " 0.020750307603143642,\n", + " 0.018948478052878526,\n", + " 0.020072371155766183,\n", + " 0.01622033783633947,\n", + " 0.017373402331382154,\n", + " 0.016252908387878462,\n", + " 0.01524356132103849,\n", + " 0.01667942963435906,\n", + " 0.018168945372750683,\n", + " 0.01717999895103886,\n", + " 0.014169907203878979,\n", + " 0.016110433558226556,\n", + " 0.016911989347595968,\n", + " 0.016918611139976197,\n", + " 0.015861555493228152,\n", + " 0.015886593668532623,\n", + " 0.019979591242851674,\n", + " 0.016244876500421668,\n", + " 0.01661257430438337,\n", + " 0.018034229228481857,\n", + " 0.016888369906079337,\n", + " 0.019784283509299783,\n", + " 0.017430895423644593,\n", + " 0.01837931725406791,\n", + " 0.020167696108900457,\n", + " 0.018854506971541146,\n", + " 0.018470328625503712,\n", + " 0.01825374339010452,\n", + " 0.018367022921516846,\n", + " 0.019696022561432547,\n", + " 0.021787655114860333,\n", + " 0.019540489850477488,\n", + " 0.01965555861963958,\n", + " 0.01910709823166229,\n", + " 0.021611410783969,\n", + " 0.018934332443253765,\n", + " 0.020873204793502076,\n", + " 0.017905055792117153,\n", + " 0.02131898649672026,\n", + " 0.020336371463632846,\n", + " 0.020498945056803664,\n", + " 0.018874717205044023,\n", + " 0.019394350497066642,\n", + " 0.018721171379744606,\n", + " 0.014928025292856556,\n", + " 0.01619391880665169,\n", + " 0.02007720814287578,\n", + " 0.01824893872654071,\n", + " 0.02001883230695936,\n", + " 0.01773258950644646,\n", + " 0.01817986292463314,\n", + " 0.019548768034712283,\n", + " 0.016988870253343002,\n", + " 0.02043270049790577,\n", + " 0.01534469283520995,\n", + " 0.017446239863274676,\n", + " 0.01712435226787648,\n", + " 0.017898903855360915]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf_kuhn_result = tf_main(game, \n", + " env_configs,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.8/dist-packages/torch/autograd/__init__.py:130: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n", + " Variable._execution_engine.run_backward(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Losses: [(tensor(0.5617), tensor(1.2384)), (None, tensor(1.0013))]\n", + "[10000] Exploitability AVG 0.43360760977233226\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3933), tensor(1.1907)), (tensor(0.4485), tensor(0.7598))]\n", + "[20000] Exploitability AVG 0.24847993754583325\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4599), tensor(1.1885)), (tensor(0.2461), tensor(1.1361))]\n", + "[30000] Exploitability AVG 0.20212825474698204\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4765), tensor(1.1473)), (tensor(0.3217), tensor(1.0794))]\n", + "[40000] Exploitability AVG 0.16857554855165724\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4865), tensor(1.1216)), (tensor(0.3131), tensor(1.0331))]\n", + "[50000] Exploitability AVG 0.15093939352390653\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3970), tensor(0.8246)), (tensor(0.2245), tensor(0.8587))]\n", + "[60000] Exploitability AVG 0.1405615429270749\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4925), tensor(0.9877)), (tensor(0.2709), tensor(0.9088))]\n", + "[70000] Exploitability AVG 0.1294691247494251\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4111), tensor(0.7660)), (tensor(0.2266), tensor(0.8891))]\n", + "[80000] Exploitability AVG 0.12353523661221327\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4465), tensor(0.8215)), (tensor(0.2443), tensor(0.7475))]\n", + "[90000] Exploitability AVG 0.11647317361335996\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4363), tensor(0.5620)), (tensor(0.1644), tensor(0.9307))]\n", + "[100000] Exploitability AVG 0.1232920439392188\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4015), tensor(1.1167)), (tensor(0.1705), tensor(0.9111))]\n", + "[110000] Exploitability AVG 0.13573878440059367\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3963), tensor(0.7951)), (tensor(0.1413), tensor(0.7840))]\n", + "[120000] Exploitability AVG 0.14339186094670014\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3841), tensor(1.1934)), (tensor(0.2165), tensor(0.9172))]\n", + "[130000] Exploitability AVG 0.14240849260532548\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3377), tensor(0.9199)), (tensor(0.1804), tensor(0.7397))]\n", + "[140000] Exploitability AVG 0.13899089965857514\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3970), tensor(0.9126)), (tensor(0.1701), tensor(1.1636))]\n", + "[150000] Exploitability AVG 0.13552169628343108\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3213), tensor(0.8355)), (tensor(0.3143), tensor(0.5159))]\n", + "[160000] Exploitability AVG 0.1311198467438126\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3521), tensor(0.6676)), (tensor(0.2828), tensor(0.8525))]\n", + "[170000] Exploitability AVG 0.1281124486926545\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4046), tensor(0.4920)), (tensor(0.2603), tensor(0.6312))]\n", + "[180000] Exploitability AVG 0.12401562839008803\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3227), tensor(0.7328)), (tensor(0.2405), tensor(0.5357))]\n", + "[190000] Exploitability AVG 0.12174890183650688\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4706), tensor(0.4364)), (tensor(0.2758), tensor(0.7155))]\n", + "[200000] Exploitability AVG 0.1240955312275292\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4443), tensor(0.6499)), (tensor(0.3358), tensor(0.6157))]\n", + "[210000] Exploitability AVG 0.1254166045280308\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3667), tensor(0.6006)), (tensor(0.2728), tensor(0.5658))]\n", + "[220000] Exploitability AVG 0.1259693811789879\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2821), tensor(0.5973)), (tensor(0.2515), tensor(0.7128))]\n", + "[230000] Exploitability AVG 0.12834214679245828\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4669), tensor(0.7966)), (tensor(0.3053), tensor(0.6479))]\n", + "[240000] Exploitability AVG 0.1306657682547336\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3578), tensor(0.6380)), (tensor(0.3230), tensor(0.5557))]\n", + "[250000] Exploitability AVG 0.13143155825253564\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3651), tensor(0.5949)), (tensor(0.3606), tensor(0.4676))]\n", + "[260000] Exploitability AVG 0.1313982201346969\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3971), tensor(0.5140)), (tensor(0.4017), tensor(0.6872))]\n", + "[270000] Exploitability AVG 0.13012119969670558\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4181), tensor(0.7868)), (tensor(0.3322), tensor(0.5078))]\n", + "[280000] Exploitability AVG 0.12718786715586541\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3605), tensor(0.5238)), (tensor(0.2890), tensor(0.6905))]\n", + "[290000] Exploitability AVG 0.12445305499534892\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4015), tensor(0.4944)), (tensor(0.3007), tensor(0.4815))]\n", + "[300000] Exploitability AVG 0.11940041676270272\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4246), tensor(0.4376)), (tensor(0.3477), tensor(0.5120))]\n", + "[310000] Exploitability AVG 0.11572987157051917\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4024), tensor(0.3739)), (tensor(0.3936), tensor(0.5279))]\n", + "[320000] Exploitability AVG 0.11187472004484214\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3777), tensor(0.4618)), (tensor(0.2699), tensor(0.5996))]\n", + "[330000] Exploitability AVG 0.1078544424117018\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3586), tensor(0.6304)), (tensor(0.3800), tensor(0.6584))]\n", + "[340000] Exploitability AVG 0.10358764818095975\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3846), tensor(0.7972)), (tensor(0.2958), tensor(0.7531))]\n", + "[350000] Exploitability AVG 0.10104319488782665\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4025), tensor(0.5350)), (tensor(0.3797), tensor(0.5959))]\n", + "[360000] Exploitability AVG 0.09718208330912342\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4297), tensor(0.5537)), (tensor(0.3523), tensor(0.4762))]\n", + "[370000] Exploitability AVG 0.0945093902880978\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3822), tensor(0.3664)), (tensor(0.3810), tensor(0.5115))]\n", + "[380000] Exploitability AVG 0.09164706043036927\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3120), tensor(0.4438)), (tensor(0.2860), tensor(0.8767))]\n", + "[390000] Exploitability AVG 0.09040716014241568\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4236), tensor(0.4680)), (tensor(0.4062), tensor(0.6563))]\n", + "[400000] Exploitability AVG 0.08637893756587489\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3497), tensor(0.4361)), (tensor(0.3580), tensor(0.5708))]\n", + "[410000] Exploitability AVG 0.0836706606604124\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4258), tensor(0.4093)), (tensor(0.2951), tensor(0.4851))]\n", + "[420000] Exploitability AVG 0.08252897921824368\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4192), tensor(0.4518)), (tensor(0.2971), tensor(0.6261))]\n", + "[430000] Exploitability AVG 0.07875484089889032\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4246), tensor(0.4417)), (tensor(0.4814), tensor(0.6235))]\n", + "[440000] Exploitability AVG 0.0784630829623014\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4114), tensor(0.3553)), (tensor(0.4271), tensor(0.5310))]\n", + "[450000] Exploitability AVG 0.08089463546506201\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3882), tensor(0.3917)), (tensor(0.4318), tensor(0.4299))]\n", + "[460000] Exploitability AVG 0.07861132029247281\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3829), tensor(0.5115)), (tensor(0.2925), tensor(0.4843))]\n", + "[470000] Exploitability AVG 0.07659106513844291\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4008), tensor(0.2779)), (tensor(0.3628), tensor(0.6053))]\n", + "[480000] Exploitability AVG 0.0779396204258512\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3889), tensor(0.3158)), (tensor(0.3960), tensor(0.5375))]\n", + "[490000] Exploitability AVG 0.07897121719627065\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3406), tensor(0.3922)), (tensor(0.4421), tensor(0.6673))]\n", + "[500000] Exploitability AVG 0.07786336464270921\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3460), tensor(0.3605)), (tensor(0.3816), tensor(0.4798))]\n", + "[510000] Exploitability AVG 0.08058506620039987\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3039), tensor(0.3618)), (tensor(0.4644), tensor(0.3860))]\n", + "[520000] Exploitability AVG 0.08017760773169297\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4209), tensor(0.3732)), (tensor(0.3857), tensor(0.8149))]\n", + "[530000] Exploitability AVG 0.08097784078093948\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3539), tensor(0.3065)), (tensor(0.3446), tensor(0.4404))]\n", + "[540000] Exploitability AVG 0.08286287203951129\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3614), tensor(0.3005)), (tensor(0.3631), tensor(0.5694))]\n", + "[550000] Exploitability AVG 0.08338122754636149\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3235), tensor(0.4094)), (tensor(0.3307), tensor(0.5443))]\n", + "[560000] Exploitability AVG 0.08240864943863549\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3675), tensor(0.4965)), (tensor(0.4050), tensor(0.4289))]\n", + "[570000] Exploitability AVG 0.07910950217120183\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4484), tensor(0.3713)), (tensor(0.4020), tensor(0.5684))]\n", + "[580000] Exploitability AVG 0.07702428816453796\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3246), tensor(0.3773)), (tensor(0.3720), tensor(0.5638))]\n", + "[590000] Exploitability AVG 0.08078908892707529\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3395), tensor(0.3641)), (tensor(0.3282), tensor(0.3540))]\n", + "[600000] Exploitability AVG 0.076801881575844\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2498), tensor(0.2638)), (tensor(0.3486), tensor(0.5069))]\n", + "[610000] Exploitability AVG 0.07418466776362032\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3800), tensor(0.2838)), (tensor(0.4196), tensor(0.6305))]\n", + "[620000] Exploitability AVG 0.07223547301108915\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3596), tensor(0.4201)), (tensor(0.3986), tensor(0.6731))]\n", + "[630000] Exploitability AVG 0.07415642818525991\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2746), tensor(0.4550)), (tensor(0.4979), tensor(0.4786))]\n", + "[640000] Exploitability AVG 0.07300951887733598\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4202), tensor(0.3822)), (tensor(0.4366), tensor(0.5048))]\n", + "[650000] Exploitability AVG 0.07085777250165912\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3378), tensor(0.2800)), (tensor(0.4472), tensor(0.6989))]\n", + "[660000] Exploitability AVG 0.07343081060256212\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3417), tensor(0.4027)), (tensor(0.4300), tensor(0.4229))]\n", + "[670000] Exploitability AVG 0.07011704960560253\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2984), tensor(0.3410)), (tensor(0.4031), tensor(0.6375))]\n", + "[680000] Exploitability AVG 0.06734571360889927\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3303), tensor(0.5154)), (tensor(0.3837), tensor(0.6855))]\n", + "[690000] Exploitability AVG 0.06794613709615879\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3435), tensor(0.4397)), (tensor(0.4821), tensor(0.7086))]\n", + "[700000] Exploitability AVG 0.06891893858672579\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3113), tensor(0.4772)), (tensor(0.4545), tensor(0.5557))]\n", + "[710000] Exploitability AVG 0.06974404369768161\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3977), tensor(0.3678)), (tensor(0.5068), tensor(0.5105))]\n", + "[720000] Exploitability AVG 0.06959025790053802\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3933), tensor(0.3503)), (tensor(0.2346), tensor(0.5392))]\n", + "[730000] Exploitability AVG 0.06912605047622833\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4282), tensor(0.3487)), (tensor(0.4061), tensor(0.4767))]\n", + "[740000] Exploitability AVG 0.06607593944739029\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3782), tensor(0.3949)), (tensor(0.3952), tensor(0.7358))]\n", + "[750000] Exploitability AVG 0.0734710028480591\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4285), tensor(0.3623)), (tensor(0.3968), tensor(0.5685))]\n", + "[760000] Exploitability AVG 0.08031856049244379\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3791), tensor(0.5355)), (tensor(0.3814), tensor(0.7640))]\n", + "[770000] Exploitability AVG 0.08192315228273192\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3515), tensor(0.5106)), (tensor(0.4115), tensor(0.6756))]\n", + "[780000] Exploitability AVG 0.08267600396333535\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3752), tensor(0.4961)), (tensor(0.3578), tensor(0.8630))]\n", + "[790000] Exploitability AVG 0.08708918652251074\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4326), tensor(0.2411)), (tensor(0.4136), tensor(0.6747))]\n", + "[800000] Exploitability AVG 0.08418522793657851\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5009), tensor(0.6107)), (tensor(0.4966), tensor(0.6741))]\n", + "[810000] Exploitability AVG 0.09198097764487115\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3977), tensor(0.5192)), (tensor(0.3757), tensor(0.6484))]\n", + "[820000] Exploitability AVG 0.08949884199838473\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4188), tensor(0.3016)), (tensor(0.4190), tensor(0.6887))]\n", + "[830000] Exploitability AVG 0.08975660242115704\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4171), tensor(0.4890)), (tensor(0.4350), tensor(0.7435))]\n", + "[840000] Exploitability AVG 0.08916363917615955\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4254), tensor(0.5706)), (tensor(0.3846), tensor(0.7005))]\n", + "[850000] Exploitability AVG 0.08630126887122891\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4875), tensor(0.6870)), (tensor(0.4259), tensor(0.7412))]\n", + "[860000] Exploitability AVG 0.08470360938219801\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4093), tensor(0.4283)), (tensor(0.3686), tensor(0.7872))]\n", + "[870000] Exploitability AVG 0.08645018090740203\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4295), tensor(0.6508)), (tensor(0.3941), tensor(0.7499))]\n", + "[880000] Exploitability AVG 0.08473272285189334\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3912), tensor(0.4092)), (tensor(0.4177), tensor(0.8367))]\n", + "[890000] Exploitability AVG 0.08968803126206024\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4053), tensor(0.5942)), (tensor(0.4381), tensor(0.8564))]\n", + "[900000] Exploitability AVG 0.08461936416279336\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4517), tensor(0.5028)), (tensor(0.4408), tensor(0.8134))]\n", + "[910000] Exploitability AVG 0.08173108498024514\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4815), tensor(0.4530)), (tensor(0.4045), tensor(0.5977))]\n", + "[920000] Exploitability AVG 0.08369104297142613\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4512), tensor(0.4513)), (tensor(0.2767), tensor(0.8015))]\n", + "[930000] Exploitability AVG 0.08363238465044989\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3593), tensor(0.6116)), (tensor(0.3753), tensor(0.6729))]\n", + "[940000] Exploitability AVG 0.0837804720222089\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4548), tensor(0.8085)), (tensor(0.5002), tensor(0.9066))]\n", + "[950000] Exploitability AVG 0.07893956613319508\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4268), tensor(0.4469)), (tensor(0.4050), tensor(0.7064))]\n", + "[960000] Exploitability AVG 0.07880904380414083\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4637), tensor(0.4897)), (tensor(0.3782), tensor(0.7337))]\n", + "[970000] Exploitability AVG 0.07956210557773066\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4537), tensor(0.7425)), (tensor(0.3851), tensor(0.7627))]\n", + "[980000] Exploitability AVG 0.08087575794167975\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3431), tensor(0.3741)), (tensor(0.4242), tensor(1.0068))]\n", + "[990000] Exploitability AVG 0.08209910522505803\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4207), tensor(0.5037)), (tensor(0.3703), tensor(0.8976))]\n", + "[1000000] Exploitability AVG 0.08132648134238146\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4554), tensor(0.5738)), (tensor(0.5110), tensor(0.8645))]\n", + "[1010000] Exploitability AVG 0.08503230281195456\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3511), tensor(0.6016)), (tensor(0.4286), tensor(0.7328))]\n", + "[1020000] Exploitability AVG 0.07817406446798755\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3240), tensor(0.7838)), (tensor(0.3747), tensor(0.7758))]\n", + "[1030000] Exploitability AVG 0.07703540740425208\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3935), tensor(0.5887)), (tensor(0.5226), tensor(1.0334))]\n", + "[1040000] Exploitability AVG 0.07771381279863096\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4174), tensor(0.4917)), (tensor(0.4157), tensor(1.0324))]\n", + "[1050000] Exploitability AVG 0.08006622550401707\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4183), tensor(0.5819)), (tensor(0.3288), tensor(0.8727))]\n", + "[1060000] Exploitability AVG 0.07616542909459811\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4041), tensor(0.7616)), (tensor(0.4054), tensor(0.9392))]\n", + "[1070000] Exploitability AVG 0.07490654630323199\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3292), tensor(0.5511)), (tensor(0.3617), tensor(0.7368))]\n", + "[1080000] Exploitability AVG 0.07141872608238131\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3528), tensor(0.4862)), (tensor(0.3584), tensor(0.8978))]\n", + "[1090000] Exploitability AVG 0.07401281870559623\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4252), tensor(0.5600)), (tensor(0.3610), tensor(0.9986))]\n", + "[1100000] Exploitability AVG 0.0754014175783275\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2706), tensor(0.3984)), (tensor(0.4349), tensor(0.7118))]\n", + "[1110000] Exploitability AVG 0.07069118499082952\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3937), tensor(0.4027)), (tensor(0.3178), tensor(0.7454))]\n", + "[1120000] Exploitability AVG 0.06790965190532028\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3438), tensor(0.7252)), (tensor(0.3973), tensor(0.5299))]\n", + "[1130000] Exploitability AVG 0.06673083601309829\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4622), tensor(0.7772)), (tensor(0.4128), tensor(0.5674))]\n", + "[1140000] Exploitability AVG 0.06641216708558975\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3198), tensor(0.7090)), (tensor(0.3340), tensor(0.7124))]\n", + "[1150000] Exploitability AVG 0.06713674952785745\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3580), tensor(0.5256)), (tensor(0.4204), tensor(0.6722))]\n", + "[1160000] Exploitability AVG 0.06582531833165697\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3488), tensor(0.7212)), (tensor(0.3974), tensor(0.7204))]\n", + "[1170000] Exploitability AVG 0.06598452473248859\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3729), tensor(0.5207)), (tensor(0.3582), tensor(0.6201))]\n", + "[1180000] Exploitability AVG 0.06089980970204942\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3641), tensor(0.6651)), (tensor(0.3528), tensor(0.7199))]\n", + "[1190000] Exploitability AVG 0.059435436543711695\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3990), tensor(0.8886)), (tensor(0.3687), tensor(0.7015))]\n", + "[1200000] Exploitability AVG 0.06300410435764975\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4243), tensor(0.6363)), (tensor(0.4236), tensor(0.7584))]\n", + "[1210000] Exploitability AVG 0.0625689386952285\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4604), tensor(0.5683)), (tensor(0.4161), tensor(0.7404))]\n", + "[1220000] Exploitability AVG 0.05727197465656633\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4258), tensor(0.6565)), (tensor(0.4147), tensor(0.7638))]\n", + "[1230000] Exploitability AVG 0.06066553844988115\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3463), tensor(0.6251)), (tensor(0.4102), tensor(0.6956))]\n", + "[1240000] Exploitability AVG 0.05875223009509098\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3738), tensor(0.5999)), (tensor(0.4421), tensor(0.7080))]\n", + "[1250000] Exploitability AVG 0.0591512354609188\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3897), tensor(0.7772)), (tensor(0.3366), tensor(0.7217))]\n", + "[1260000] Exploitability AVG 0.05756266692938833\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3788), tensor(0.5485)), (tensor(0.3741), tensor(0.8414))]\n", + "[1270000] Exploitability AVG 0.05860752690010504\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4514), tensor(0.6886)), (tensor(0.4220), tensor(0.6514))]\n", + "[1280000] Exploitability AVG 0.055941788437478046\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4186), tensor(0.7316)), (tensor(0.3729), tensor(0.4838))]\n", + "[1290000] Exploitability AVG 0.05479102123194027\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3420), tensor(0.5531)), (tensor(0.3379), tensor(0.7351))]\n", + "[1300000] Exploitability AVG 0.05388859654788222\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3377), tensor(0.4890)), (tensor(0.3386), tensor(0.7816))]\n", + "[1310000] Exploitability AVG 0.05318028974511507\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4003), tensor(0.6782)), (tensor(0.4799), tensor(0.6077))]\n", + "[1320000] Exploitability AVG 0.05369673495142163\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3806), tensor(0.5932)), (tensor(0.3416), tensor(0.6167))]\n", + "[1330000] Exploitability AVG 0.056876447226549\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4297), tensor(0.4890)), (tensor(0.4849), tensor(0.7438))]\n", + "[1340000] Exploitability AVG 0.05265122975658748\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3853), tensor(0.4934)), (tensor(0.2513), tensor(0.6085))]\n", + "[1350000] Exploitability AVG 0.054076250578127344\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3895), tensor(0.6986)), (tensor(0.4048), tensor(0.6211))]\n", + "[1360000] Exploitability AVG 0.054303770909174515\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3726), tensor(0.5673)), (tensor(0.3541), tensor(0.4419))]\n", + "[1370000] Exploitability AVG 0.052591564672165136\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4242), tensor(0.4943)), (tensor(0.3906), tensor(0.8334))]\n", + "[1380000] Exploitability AVG 0.04952984181538464\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4134), tensor(0.6572)), (tensor(0.4389), tensor(0.5610))]\n", + "[1390000] Exploitability AVG 0.05405613568042206\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4336), tensor(0.7102)), (tensor(0.3184), tensor(0.9579))]\n", + "[1400000] Exploitability AVG 0.04951620347516003\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4908), tensor(0.6272)), (tensor(0.3794), tensor(0.6340))]\n", + "[1410000] Exploitability AVG 0.05170689896869221\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2799), tensor(0.6782)), (tensor(0.3503), tensor(0.6603))]\n", + "[1420000] Exploitability AVG 0.051677095144450846\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3677), tensor(0.6417)), (tensor(0.3760), tensor(0.6529))]\n", + "[1430000] Exploitability AVG 0.05115030829705855\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3134), tensor(0.2873)), (tensor(0.3101), tensor(0.4847))]\n", + "[1440000] Exploitability AVG 0.04865786442362896\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4975), tensor(0.8507)), (tensor(0.3335), tensor(0.7662))]\n", + "[1450000] Exploitability AVG 0.0515626341756317\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4249), tensor(0.3697)), (tensor(0.3495), tensor(0.7194))]\n", + "[1460000] Exploitability AVG 0.04597114187720136\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3648), tensor(0.6388)), (tensor(0.3507), tensor(0.6714))]\n", + "[1470000] Exploitability AVG 0.045788884240381605\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3713), tensor(0.4208)), (tensor(0.3784), tensor(0.5076))]\n", + "[1480000] Exploitability AVG 0.04993328302018954\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4020), tensor(0.4952)), (tensor(0.3242), tensor(0.7236))]\n", + "[1490000] Exploitability AVG 0.04701021820911136\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3639), tensor(0.6737)), (tensor(0.4033), tensor(0.6060))]\n", + "[1500000] Exploitability AVG 0.047488750797572865\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3505), tensor(0.5921)), (tensor(0.4726), tensor(0.4826))]\n", + "[1510000] Exploitability AVG 0.045587313748810976\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3504), tensor(0.5503)), (tensor(0.3188), tensor(0.6447))]\n", + "[1520000] Exploitability AVG 0.04484532722577156\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3579), tensor(0.3807)), (tensor(0.3520), tensor(0.5276))]\n", + "[1530000] Exploitability AVG 0.04531227990606543\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3553), tensor(0.5467)), (tensor(0.3754), tensor(0.5651))]\n", + "[1540000] Exploitability AVG 0.04441547314052871\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3109), tensor(0.4062)), (tensor(0.3415), tensor(0.5116))]\n", + "[1550000] Exploitability AVG 0.041771529755133624\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4309), tensor(0.3827)), (tensor(0.5139), tensor(0.6435))]\n", + "[1560000] Exploitability AVG 0.04661851857309837\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4476), tensor(0.7518)), (tensor(0.3969), tensor(0.6489))]\n", + "[1570000] Exploitability AVG 0.043218003287104095\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4191), tensor(0.7098)), (tensor(0.4589), tensor(0.7429))]\n", + "[1580000] Exploitability AVG 0.04291629204060657\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3879), tensor(0.5745)), (tensor(0.4193), tensor(0.4596))]\n", + "[1590000] Exploitability AVG 0.04004542061016397\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3223), tensor(0.5518)), (tensor(0.2810), tensor(0.6055))]\n", + "[1600000] Exploitability AVG 0.04184367544476339\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3189), tensor(0.6197)), (tensor(0.3857), tensor(0.4696))]\n", + "[1610000] Exploitability AVG 0.03852252588169325\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4658), tensor(0.8411)), (tensor(0.3216), tensor(0.7314))]\n", + "[1620000] Exploitability AVG 0.04049332539563036\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4809), tensor(0.4893)), (tensor(0.4238), tensor(0.5391))]\n", + "[1630000] Exploitability AVG 0.04027982045376177\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3811), tensor(0.7558)), (tensor(0.5191), tensor(0.7393))]\n", + "[1640000] Exploitability AVG 0.04189048620746311\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3523), tensor(0.5060)), (tensor(0.2649), tensor(0.4363))]\n", + "[1650000] Exploitability AVG 0.03983469161069905\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3527), tensor(0.3634)), (tensor(0.3956), tensor(0.4747))]\n", + "[1660000] Exploitability AVG 0.03931548090824466\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3792), tensor(0.5759)), (tensor(0.3370), tensor(0.7024))]\n", + "[1670000] Exploitability AVG 0.04068559242383468\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4025), tensor(0.3502)), (tensor(0.2579), tensor(0.5738))]\n", + "[1680000] Exploitability AVG 0.04074387081065195\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4243), tensor(0.5397)), (tensor(0.4018), tensor(0.5581))]\n", + "[1690000] Exploitability AVG 0.040161238395097415\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4015), tensor(0.6107)), (tensor(0.3374), tensor(0.7067))]\n", + "[1700000] Exploitability AVG 0.03921280686511541\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4469), tensor(0.4640)), (tensor(0.4740), tensor(0.6388))]\n", + "[1710000] Exploitability AVG 0.040116030607185466\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3541), tensor(0.4486)), (tensor(0.2786), tensor(0.4513))]\n", + "[1720000] Exploitability AVG 0.04278309803768987\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3670), tensor(0.5252)), (tensor(0.2966), tensor(0.6068))]\n", + "[1730000] Exploitability AVG 0.03978450110850876\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4172), tensor(0.9239)), (tensor(0.4663), tensor(0.5822))]\n", + "[1740000] Exploitability AVG 0.042412997566294186\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4117), tensor(0.5945)), (tensor(0.3399), tensor(0.5413))]\n", + "[1750000] Exploitability AVG 0.037822143090520344\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3458), tensor(0.5247)), (tensor(0.3513), tensor(0.6219))]\n", + "[1760000] Exploitability AVG 0.03808836029091858\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3765), tensor(0.6410)), (tensor(0.3842), tensor(0.5667))]\n", + "[1770000] Exploitability AVG 0.03604818643633653\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4220), tensor(0.3814)), (tensor(0.3496), tensor(0.4355))]\n", + "[1780000] Exploitability AVG 0.0375807307817419\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3579), tensor(0.6180)), (tensor(0.3113), tensor(0.4865))]\n", + "[1790000] Exploitability AVG 0.03763102345628172\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3678), tensor(0.5813)), (tensor(0.4297), tensor(0.6624))]\n", + "[1800000] Exploitability AVG 0.036444810971938246\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3675), tensor(0.5595)), (tensor(0.3252), tensor(0.5444))]\n", + "[1810000] Exploitability AVG 0.036534741309934116\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3838), tensor(0.5547)), (tensor(0.3517), tensor(0.5694))]\n", + "[1820000] Exploitability AVG 0.03543566918231755\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4754), tensor(0.6316)), (tensor(0.3603), tensor(0.5703))]\n", + "[1830000] Exploitability AVG 0.03407305949882461\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3994), tensor(0.5082)), (tensor(0.3381), tensor(0.5491))]\n", + "[1840000] Exploitability AVG 0.033185615799471296\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4102), tensor(0.5825)), (tensor(0.3652), tensor(0.5195))]\n", + "[1850000] Exploitability AVG 0.034837403065548084\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3701), tensor(0.4704)), (tensor(0.3966), tensor(0.4434))]\n", + "[1860000] Exploitability AVG 0.03454685022740092\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4468), tensor(0.6368)), (tensor(0.2935), tensor(0.5337))]\n", + "[1870000] Exploitability AVG 0.03164810589351297\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4230), tensor(0.5551)), (tensor(0.4416), tensor(0.6778))]\n", + "[1880000] Exploitability AVG 0.03414953169347454\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3092), tensor(0.3547)), (tensor(0.3736), tensor(0.5461))]\n", + "[1890000] Exploitability AVG 0.032492169852572195\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3306), tensor(0.5233)), (tensor(0.3079), tensor(0.7119))]\n", + "[1900000] Exploitability AVG 0.03492351048498499\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3372), tensor(0.4695)), (tensor(0.2809), tensor(0.6085))]\n", + "[1910000] Exploitability AVG 0.02920019675656324\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3614), tensor(0.4667)), (tensor(0.3204), tensor(0.5959))]\n", + "[1920000] Exploitability AVG 0.030073471095696275\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3976), tensor(0.6265)), (tensor(0.3710), tensor(0.4609))]\n", + "[1930000] Exploitability AVG 0.030795867788258907\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4124), tensor(0.6946)), (tensor(0.2949), tensor(0.6526))]\n", + "[1940000] Exploitability AVG 0.03265946493342006\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3958), tensor(0.4310)), (tensor(0.3680), tensor(0.5895))]\n", + "[1950000] Exploitability AVG 0.033916676327968964\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4152), tensor(0.3779)), (tensor(0.3603), tensor(0.4958))]\n", + "[1960000] Exploitability AVG 0.03180096995453127\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4569), tensor(0.6108)), (tensor(0.4204), tensor(0.6166))]\n", + "[1970000] Exploitability AVG 0.03303150101101421\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4002), tensor(0.4913)), (tensor(0.3464), tensor(0.3709))]\n", + "[1980000] Exploitability AVG 0.0260980560218885\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4075), tensor(0.6243)), (tensor(0.3433), tensor(0.5540))]\n", + "[1990000] Exploitability AVG 0.030362039397051954\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3843), tensor(0.4094)), (tensor(0.3356), tensor(0.5296))]\n", + "[2000000] Exploitability AVG 0.02580402572335483\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4389), tensor(0.5677)), (tensor(0.4065), tensor(0.4047))]\n", + "[2010000] Exploitability AVG 0.029126794448983417\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3831), tensor(0.6793)), (tensor(0.3284), tensor(0.8041))]\n", + "[2020000] Exploitability AVG 0.03043367926715243\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3577), tensor(0.5169)), (tensor(0.4356), tensor(0.5572))]\n", + "[2030000] Exploitability AVG 0.026322360400955674\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3572), tensor(0.8518)), (tensor(0.4086), tensor(0.4817))]\n", + "[2040000] Exploitability AVG 0.02931724458151644\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3029), tensor(0.4532)), (tensor(0.3473), tensor(0.4599))]\n", + "[2050000] Exploitability AVG 0.0222741672152221\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4292), tensor(0.5133)), (tensor(0.2492), tensor(0.6043))]\n", + "[2060000] Exploitability AVG 0.025832072570241615\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3871), tensor(0.5093)), (tensor(0.3447), tensor(0.6787))]\n", + "[2070000] Exploitability AVG 0.02343176032594757\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3642), tensor(0.4636)), (tensor(0.3281), tensor(0.5955))]\n", + "[2080000] Exploitability AVG 0.022162447565084814\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3018), tensor(0.5602)), (tensor(0.4552), tensor(0.6612))]\n", + "[2090000] Exploitability AVG 0.025451017162971296\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3467), tensor(0.6004)), (tensor(0.4406), tensor(0.5528))]\n", + "[2100000] Exploitability AVG 0.025083265764976687\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3997), tensor(0.7116)), (tensor(0.2877), tensor(0.4371))]\n", + "[2110000] Exploitability AVG 0.023393276728879808\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3919), tensor(0.6051)), (tensor(0.3092), tensor(0.5316))]\n", + "[2120000] Exploitability AVG 0.0230838464630361\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3729), tensor(0.5205)), (tensor(0.3463), tensor(0.6366))]\n", + "[2130000] Exploitability AVG 0.0228302785278148\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3419), tensor(0.4810)), (tensor(0.4053), tensor(0.5300))]\n", + "[2140000] Exploitability AVG 0.025741539714318917\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3683), tensor(0.5392)), (tensor(0.3343), tensor(0.6475))]\n", + "[2150000] Exploitability AVG 0.024462357701041354\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4460), tensor(0.6338)), (tensor(0.3023), tensor(0.5160))]\n", + "[2160000] Exploitability AVG 0.022225538691915375\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4456), tensor(0.5661)), (tensor(0.3049), tensor(0.4931))]\n", + "[2170000] Exploitability AVG 0.021912958943342525\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3800), tensor(0.5788)), (tensor(0.3513), tensor(0.7652))]\n", + "[2180000] Exploitability AVG 0.022744310996716283\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4454), tensor(0.4942)), (tensor(0.2832), tensor(0.4071))]\n", + "[2190000] Exploitability AVG 0.021391358375225017\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4395), tensor(0.7967)), (tensor(0.3720), tensor(0.4530))]\n", + "[2200000] Exploitability AVG 0.019633380067774514\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4051), tensor(0.4590)), (tensor(0.3388), tensor(0.5634))]\n", + "[2210000] Exploitability AVG 0.018788541989070956\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4071), tensor(0.4384)), (tensor(0.3658), tensor(0.6981))]\n", + "[2220000] Exploitability AVG 0.02093171268158192\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4236), tensor(0.6637)), (tensor(0.3921), tensor(0.6431))]\n", + "[2230000] Exploitability AVG 0.021093723992809288\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3815), tensor(0.5523)), (tensor(0.4323), tensor(0.5221))]\n", + "[2240000] Exploitability AVG 0.018714704091594342\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3545), tensor(0.5282)), (tensor(0.3858), tensor(0.4508))]\n", + "[2250000] Exploitability AVG 0.01933364397497503\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3928), tensor(0.3577)), (tensor(0.3820), tensor(0.7197))]\n", + "[2260000] Exploitability AVG 0.020633415250848525\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4030), tensor(0.5697)), (tensor(0.3100), tensor(0.6342))]\n", + "[2270000] Exploitability AVG 0.018683385735314056\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4088), tensor(0.4561)), (tensor(0.2766), tensor(0.8086))]\n", + "[2280000] Exploitability AVG 0.018138049735475864\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4102), tensor(0.5425)), (tensor(0.4573), tensor(0.3978))]\n", + "[2290000] Exploitability AVG 0.017165023377770455\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4041), tensor(0.5065)), (tensor(0.3854), tensor(0.6564))]\n", + "[2300000] Exploitability AVG 0.018523783301463842\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4670), tensor(0.6273)), (tensor(0.3170), tensor(0.5633))]\n", + "[2310000] Exploitability AVG 0.016517746975265563\n", + "_____________________________________________\n", + "Losses: [(tensor(0.2930), tensor(0.4683)), (tensor(0.3357), tensor(0.6861))]\n", + "[2320000] Exploitability AVG 0.01669886839634377\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4190), tensor(0.8116)), (tensor(0.4075), tensor(0.8588))]\n", + "[2330000] Exploitability AVG 0.01647914928719635\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3593), tensor(0.6478)), (tensor(0.4878), tensor(0.5407))]\n", + "[2340000] Exploitability AVG 0.015554086566889114\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3548), tensor(0.4391)), (tensor(0.2859), tensor(0.4295))]\n", + "[2350000] Exploitability AVG 0.01597738560404452\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4562), tensor(0.6553)), (tensor(0.3443), tensor(0.5110))]\n", + "[2360000] Exploitability AVG 0.015916733028047886\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4004), tensor(0.4601)), (tensor(0.3454), tensor(0.5199))]\n", + "[2370000] Exploitability AVG 0.016055443086034543\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4309), tensor(0.7187)), (tensor(0.3584), tensor(0.3988))]\n", + "[2380000] Exploitability AVG 0.01780281759456248\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3511), tensor(0.6838)), (tensor(0.3549), tensor(0.5552))]\n", + "[2390000] Exploitability AVG 0.016218708712828045\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4296), tensor(0.4245)), (tensor(0.3562), tensor(0.6341))]\n", + "[2400000] Exploitability AVG 0.017802691576397578\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4476), tensor(0.5208)), (tensor(0.3388), tensor(0.7039))]\n", + "[2410000] Exploitability AVG 0.013508953848357319\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4301), tensor(0.5230)), (tensor(0.4661), tensor(0.5026))]\n", + "[2420000] Exploitability AVG 0.014755528011493574\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4372), tensor(0.6009)), (tensor(0.3393), tensor(0.7861))]\n", + "[2430000] Exploitability AVG 0.018553143290686525\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4286), tensor(0.6902)), (tensor(0.3051), tensor(0.6809))]\n", + "[2440000] Exploitability AVG 0.017016519431731575\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4968), tensor(0.6796)), (tensor(0.3880), tensor(0.6707))]\n", + "[2450000] Exploitability AVG 0.017095079572793576\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3801), tensor(0.6021)), (tensor(0.3143), tensor(0.6614))]\n", + "[2460000] Exploitability AVG 0.020741585257575246\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3624), tensor(0.7379)), (tensor(0.3013), tensor(0.6739))]\n", + "[2470000] Exploitability AVG 0.0186017358408144\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4692), tensor(0.4845)), (tensor(0.3789), tensor(0.6933))]\n", + "[2480000] Exploitability AVG 0.016857291040643013\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3825), tensor(0.4457)), (tensor(0.4021), tensor(0.7688))]\n", + "[2490000] Exploitability AVG 0.01581194738239189\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3480), tensor(0.5957)), (tensor(0.3243), tensor(0.5053))]\n", + "[2500000] Exploitability AVG 0.016462095098913304\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4407), tensor(0.6944)), (tensor(0.2828), tensor(0.4169))]\n", + "[2510000] Exploitability AVG 0.01610565620748919\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4406), tensor(0.8200)), (tensor(0.3810), tensor(0.5028))]\n", + "[2520000] Exploitability AVG 0.019459837788758372\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3822), tensor(0.5562)), (tensor(0.4319), tensor(0.8071))]\n", + "[2530000] Exploitability AVG 0.017826240276380606\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4499), tensor(0.6991)), (tensor(0.3387), tensor(0.7242))]\n", + "[2540000] Exploitability AVG 0.017450156843794945\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3592), tensor(0.5174)), (tensor(0.3899), tensor(0.5678))]\n", + "[2550000] Exploitability AVG 0.01930282258341648\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4345), tensor(0.5211)), (tensor(0.3158), tensor(0.7255))]\n", + "[2560000] Exploitability AVG 0.021309122619624615\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4641), tensor(0.4103)), (tensor(0.2720), tensor(0.6569))]\n", + "[2570000] Exploitability AVG 0.01909885289292787\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3737), tensor(0.5927)), (tensor(0.2621), tensor(0.5053))]\n", + "[2580000] Exploitability AVG 0.019417194896659357\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4273), tensor(0.4899)), (tensor(0.3536), tensor(0.5198))]\n", + "[2590000] Exploitability AVG 0.020013908201895914\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4147), tensor(0.5475)), (tensor(0.3833), tensor(0.5205))]\n", + "[2600000] Exploitability AVG 0.019429275237962923\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4194), tensor(0.5771)), (tensor(0.4319), tensor(0.4813))]\n", + "[2610000] Exploitability AVG 0.019685445149001224\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3853), tensor(0.6366)), (tensor(0.3604), tensor(0.4914))]\n", + "[2620000] Exploitability AVG 0.01838419029010832\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3480), tensor(0.5009)), (tensor(0.3811), tensor(0.4139))]\n", + "[2630000] Exploitability AVG 0.0176459175949622\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3650), tensor(0.4722)), (tensor(0.3035), tensor(0.6596))]\n", + "[2640000] Exploitability AVG 0.016971091994273324\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4327), tensor(0.6858)), (tensor(0.2958), tensor(0.5352))]\n", + "[2650000] Exploitability AVG 0.017541741152948043\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3904), tensor(0.5716)), (tensor(0.3200), tensor(0.4858))]\n", + "[2660000] Exploitability AVG 0.016731886920423406\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3979), tensor(0.6012)), (tensor(0.3287), tensor(0.7641))]\n", + "[2670000] Exploitability AVG 0.01808383379591874\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4666), tensor(0.6727)), (tensor(0.2967), tensor(0.6948))]\n", + "[2680000] Exploitability AVG 0.016100713641114206\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3438), tensor(0.4277)), (tensor(0.3265), tensor(0.4048))]\n", + "[2690000] Exploitability AVG 0.018846117409032775\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3772), tensor(0.3123)), (tensor(0.3492), tensor(0.6501))]\n", + "[2700000] Exploitability AVG 0.014911325239323897\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4955), tensor(0.5065)), (tensor(0.3781), tensor(0.5451))]\n", + "[2710000] Exploitability AVG 0.015246258866611961\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3631), tensor(0.6280)), (tensor(0.3289), tensor(0.5484))]\n", + "[2720000] Exploitability AVG 0.016192666972175973\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3409), tensor(0.5891)), (tensor(0.3438), tensor(0.5533))]\n", + "[2730000] Exploitability AVG 0.014383386742048088\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4584), tensor(0.5547)), (tensor(0.3006), tensor(0.6924))]\n", + "[2740000] Exploitability AVG 0.013323312414487198\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3742), tensor(0.6319)), (tensor(0.3438), tensor(0.4374))]\n", + "[2750000] Exploitability AVG 0.0184816951845799\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4075), tensor(0.6286)), (tensor(0.3922), tensor(0.4814))]\n", + "[2760000] Exploitability AVG 0.014722286465093354\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3660), tensor(0.5713)), (tensor(0.3197), tensor(0.6964))]\n", + "[2770000] Exploitability AVG 0.015530198121108035\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4022), tensor(0.5852)), (tensor(0.3744), tensor(0.6212))]\n", + "[2780000] Exploitability AVG 0.01625703220608893\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3841), tensor(0.7047)), (tensor(0.2791), tensor(0.4944))]\n", + "[2790000] Exploitability AVG 0.015654835660013605\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4286), tensor(0.5599)), (tensor(0.3314), tensor(0.7598))]\n", + "[2800000] Exploitability AVG 0.01343730087277345\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4148), tensor(0.5863)), (tensor(0.4682), tensor(0.5025))]\n", + "[2810000] Exploitability AVG 0.015231834610727024\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4488), tensor(0.6581)), (tensor(0.3380), tensor(0.5297))]\n", + "[2820000] Exploitability AVG 0.019024162240735915\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3662), tensor(0.7592)), (tensor(0.3613), tensor(0.5753))]\n", + "[2830000] Exploitability AVG 0.016046657711827617\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4628), tensor(0.5754)), (tensor(0.3329), tensor(0.4117))]\n", + "[2840000] Exploitability AVG 0.016609938503202692\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4475), tensor(0.6964)), (tensor(0.3682), tensor(0.5200))]\n", + "[2850000] Exploitability AVG 0.01831173714057166\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4491), tensor(0.5608)), (tensor(0.4532), tensor(0.6663))]\n", + "[2860000] Exploitability AVG 0.015905870053214533\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3869), tensor(0.8572)), (tensor(0.3168), tensor(0.6470))]\n", + "[2870000] Exploitability AVG 0.017702698810962814\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3185), tensor(0.5690)), (tensor(0.4040), tensor(0.8963))]\n", + "[2880000] Exploitability AVG 0.016855125244009783\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3543), tensor(0.4454)), (tensor(0.4028), tensor(0.5151))]\n", + "[2890000] Exploitability AVG 0.017209345890725952\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4069), tensor(0.5273)), (tensor(0.4027), tensor(0.6172))]\n", + "[2900000] Exploitability AVG 0.020294385674139964\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4423), tensor(0.6132)), (tensor(0.3546), tensor(0.5599))]\n", + "[2910000] Exploitability AVG 0.017391673354361953\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3939), tensor(0.6952)), (tensor(0.3142), tensor(0.7527))]\n", + "[2920000] Exploitability AVG 0.01748352211929516\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3866), tensor(0.7582)), (tensor(0.3583), tensor(0.6823))]\n", + "[2930000] Exploitability AVG 0.01859828530995042\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4350), tensor(0.5974)), (tensor(0.2717), tensor(0.6822))]\n", + "[2940000] Exploitability AVG 0.018402718202180895\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4956), tensor(0.6331)), (tensor(0.3572), tensor(0.5940))]\n", + "[2950000] Exploitability AVG 0.016206905211255562\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3647), tensor(0.4823)), (tensor(0.3137), tensor(0.5425))]\n", + "[2960000] Exploitability AVG 0.019286710911544963\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5160), tensor(0.6893)), (tensor(0.4441), tensor(0.6221))]\n", + "[2970000] Exploitability AVG 0.016659304559877103\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4204), tensor(0.6433)), (tensor(0.3876), tensor(0.7069))]\n", + "[2980000] Exploitability AVG 0.018481970706874074\n", + "_____________________________________________\n", + "Losses: [(tensor(0.3282), tensor(0.4546)), (tensor(0.3089), tensor(0.6041))]\n", + "[2990000] Exploitability AVG 0.014478293336704112\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4022), tensor(0.6153)), (tensor(0.3736), tensor(0.7058))]\n", + "[3000000] Exploitability AVG 0.015763380753411382\n", + "_____________________________________________\n" + ] + }, + { + "data": { + "text/plain": [ + "[0.43360760977233226,\n", + " 0.24847993754583325,\n", + " 0.20212825474698204,\n", + " 0.16857554855165724,\n", + " 0.15093939352390653,\n", + " 0.1405615429270749,\n", + " 0.1294691247494251,\n", + " 0.12353523661221327,\n", + " 0.11647317361335996,\n", + " 0.1232920439392188,\n", + " 0.13573878440059367,\n", + " 0.14339186094670014,\n", + " 0.14240849260532548,\n", + " 0.13899089965857514,\n", + " 0.13552169628343108,\n", + " 0.1311198467438126,\n", + " 0.1281124486926545,\n", + " 0.12401562839008803,\n", + " 0.12174890183650688,\n", + " 0.1240955312275292,\n", + " 0.1254166045280308,\n", + " 0.1259693811789879,\n", + " 0.12834214679245828,\n", + " 0.1306657682547336,\n", + " 0.13143155825253564,\n", + " 0.1313982201346969,\n", + " 0.13012119969670558,\n", + " 0.12718786715586541,\n", + " 0.12445305499534892,\n", + " 0.11940041676270272,\n", + " 0.11572987157051917,\n", + " 0.11187472004484214,\n", + " 0.1078544424117018,\n", + " 0.10358764818095975,\n", + " 0.10104319488782665,\n", + " 0.09718208330912342,\n", + " 0.0945093902880978,\n", + " 0.09164706043036927,\n", + " 0.09040716014241568,\n", + " 0.08637893756587489,\n", + " 0.0836706606604124,\n", + " 0.08252897921824368,\n", + " 0.07875484089889032,\n", + " 0.0784630829623014,\n", + " 0.08089463546506201,\n", + " 0.07861132029247281,\n", + " 0.07659106513844291,\n", + " 0.0779396204258512,\n", + " 0.07897121719627065,\n", + " 0.07786336464270921,\n", + " 0.08058506620039987,\n", + " 0.08017760773169297,\n", + " 0.08097784078093948,\n", + " 0.08286287203951129,\n", + " 0.08338122754636149,\n", + " 0.08240864943863549,\n", + " 0.07910950217120183,\n", + " 0.07702428816453796,\n", + " 0.08078908892707529,\n", + " 0.076801881575844,\n", + " 0.07418466776362032,\n", + " 0.07223547301108915,\n", + " 0.07415642818525991,\n", + " 0.07300951887733598,\n", + " 0.07085777250165912,\n", + " 0.07343081060256212,\n", + " 0.07011704960560253,\n", + " 0.06734571360889927,\n", + " 0.06794613709615879,\n", + " 0.06891893858672579,\n", + " 0.06974404369768161,\n", + " 0.06959025790053802,\n", + " 0.06912605047622833,\n", + " 0.06607593944739029,\n", + " 0.0734710028480591,\n", + " 0.08031856049244379,\n", + " 0.08192315228273192,\n", + " 0.08267600396333535,\n", + " 0.08708918652251074,\n", + " 0.08418522793657851,\n", + " 0.09198097764487115,\n", + " 0.08949884199838473,\n", + " 0.08975660242115704,\n", + " 0.08916363917615955,\n", + " 0.08630126887122891,\n", + " 0.08470360938219801,\n", + " 0.08645018090740203,\n", + " 0.08473272285189334,\n", + " 0.08968803126206024,\n", + " 0.08461936416279336,\n", + " 0.08173108498024514,\n", + " 0.08369104297142613,\n", + " 0.08363238465044989,\n", + " 0.0837804720222089,\n", + " 0.07893956613319508,\n", + " 0.07880904380414083,\n", + " 0.07956210557773066,\n", + " 0.08087575794167975,\n", + " 0.08209910522505803,\n", + " 0.08132648134238146,\n", + " 0.08503230281195456,\n", + " 0.07817406446798755,\n", + " 0.07703540740425208,\n", + " 0.07771381279863096,\n", + " 0.08006622550401707,\n", + " 0.07616542909459811,\n", + " 0.07490654630323199,\n", + " 0.07141872608238131,\n", + " 0.07401281870559623,\n", + " 0.0754014175783275,\n", + " 0.07069118499082952,\n", + " 0.06790965190532028,\n", + " 0.06673083601309829,\n", + " 0.06641216708558975,\n", + " 0.06713674952785745,\n", + " 0.06582531833165697,\n", + " 0.06598452473248859,\n", + " 0.06089980970204942,\n", + " 0.059435436543711695,\n", + " 0.06300410435764975,\n", + " 0.0625689386952285,\n", + " 0.05727197465656633,\n", + " 0.06066553844988115,\n", + " 0.05875223009509098,\n", + " 0.0591512354609188,\n", + " 0.05756266692938833,\n", + " 0.05860752690010504,\n", + " 0.055941788437478046,\n", + " 0.05479102123194027,\n", + " 0.05388859654788222,\n", + " 0.05318028974511507,\n", + " 0.05369673495142163,\n", + " 0.056876447226549,\n", + " 0.05265122975658748,\n", + " 0.054076250578127344,\n", + " 0.054303770909174515,\n", + " 0.052591564672165136,\n", + " 0.04952984181538464,\n", + " 0.05405613568042206,\n", + " 0.04951620347516003,\n", + " 0.05170689896869221,\n", + " 0.051677095144450846,\n", + " 0.05115030829705855,\n", + " 0.04865786442362896,\n", + " 0.0515626341756317,\n", + " 0.04597114187720136,\n", + " 0.045788884240381605,\n", + " 0.04993328302018954,\n", + " 0.04701021820911136,\n", + " 0.047488750797572865,\n", + " 0.045587313748810976,\n", + " 0.04484532722577156,\n", + " 0.04531227990606543,\n", + " 0.04441547314052871,\n", + " 0.041771529755133624,\n", + " 0.04661851857309837,\n", + " 0.043218003287104095,\n", + " 0.04291629204060657,\n", + " 0.04004542061016397,\n", + " 0.04184367544476339,\n", + " 0.03852252588169325,\n", + " 0.04049332539563036,\n", + " 0.04027982045376177,\n", + " 0.04189048620746311,\n", + " 0.03983469161069905,\n", + " 0.03931548090824466,\n", + " 0.04068559242383468,\n", + " 0.04074387081065195,\n", + " 0.040161238395097415,\n", + " 0.03921280686511541,\n", + " 0.040116030607185466,\n", + " 0.04278309803768987,\n", + " 0.03978450110850876,\n", + " 0.042412997566294186,\n", + " 0.037822143090520344,\n", + " 0.03808836029091858,\n", + " 0.03604818643633653,\n", + " 0.0375807307817419,\n", + " 0.03763102345628172,\n", + " 0.036444810971938246,\n", + " 0.036534741309934116,\n", + " 0.03543566918231755,\n", + " 0.03407305949882461,\n", + " 0.033185615799471296,\n", + " 0.034837403065548084,\n", + " 0.03454685022740092,\n", + " 0.03164810589351297,\n", + " 0.03414953169347454,\n", + " 0.032492169852572195,\n", + " 0.03492351048498499,\n", + " 0.02920019675656324,\n", + " 0.030073471095696275,\n", + " 0.030795867788258907,\n", + " 0.03265946493342006,\n", + " 0.033916676327968964,\n", + " 0.03180096995453127,\n", + " 0.03303150101101421,\n", + " 0.0260980560218885,\n", + " 0.030362039397051954,\n", + " 0.02580402572335483,\n", + " 0.029126794448983417,\n", + " 0.03043367926715243,\n", + " 0.026322360400955674,\n", + " 0.02931724458151644,\n", + " 0.0222741672152221,\n", + " 0.025832072570241615,\n", + " 0.02343176032594757,\n", + " 0.022162447565084814,\n", + " 0.025451017162971296,\n", + " 0.025083265764976687,\n", + " 0.023393276728879808,\n", + " 0.0230838464630361,\n", + " 0.0228302785278148,\n", + " 0.025741539714318917,\n", + " 0.024462357701041354,\n", + " 0.022225538691915375,\n", + " 0.021912958943342525,\n", + " 0.022744310996716283,\n", + " 0.021391358375225017,\n", + " 0.019633380067774514,\n", + " 0.018788541989070956,\n", + " 0.02093171268158192,\n", + " 0.021093723992809288,\n", + " 0.018714704091594342,\n", + " 0.01933364397497503,\n", + " 0.020633415250848525,\n", + " 0.018683385735314056,\n", + " 0.018138049735475864,\n", + " 0.017165023377770455,\n", + " 0.018523783301463842,\n", + " 0.016517746975265563,\n", + " 0.01669886839634377,\n", + " 0.01647914928719635,\n", + " 0.015554086566889114,\n", + " 0.01597738560404452,\n", + " 0.015916733028047886,\n", + " 0.016055443086034543,\n", + " 0.01780281759456248,\n", + " 0.016218708712828045,\n", + " 0.017802691576397578,\n", + " 0.013508953848357319,\n", + " 0.014755528011493574,\n", + " 0.018553143290686525,\n", + " 0.017016519431731575,\n", + " 0.017095079572793576,\n", + " 0.020741585257575246,\n", + " 0.0186017358408144,\n", + " 0.016857291040643013,\n", + " 0.01581194738239189,\n", + " 0.016462095098913304,\n", + " 0.01610565620748919,\n", + " 0.019459837788758372,\n", + " 0.017826240276380606,\n", + " 0.017450156843794945,\n", + " 0.01930282258341648,\n", + " 0.021309122619624615,\n", + " 0.01909885289292787,\n", + " 0.019417194896659357,\n", + " 0.020013908201895914,\n", + " 0.019429275237962923,\n", + " 0.019685445149001224,\n", + " 0.01838419029010832,\n", + " 0.0176459175949622,\n", + " 0.016971091994273324,\n", + " 0.017541741152948043,\n", + " 0.016731886920423406,\n", + " 0.01808383379591874,\n", + " 0.016100713641114206,\n", + " 0.018846117409032775,\n", + " 0.014911325239323897,\n", + " 0.015246258866611961,\n", + " 0.016192666972175973,\n", + " 0.014383386742048088,\n", + " 0.013323312414487198,\n", + " 0.0184816951845799,\n", + " 0.014722286465093354,\n", + " 0.015530198121108035,\n", + " 0.01625703220608893,\n", + " 0.015654835660013605,\n", + " 0.01343730087277345,\n", + " 0.015231834610727024,\n", + " 0.019024162240735915,\n", + " 0.016046657711827617,\n", + " 0.016609938503202692,\n", + " 0.01831173714057166,\n", + " 0.015905870053214533,\n", + " 0.017702698810962814,\n", + " 0.016855125244009783,\n", + " 0.017209345890725952,\n", + " 0.020294385674139964,\n", + " 0.017391673354361953,\n", + " 0.01748352211929516,\n", + " 0.01859828530995042,\n", + " 0.018402718202180895,\n", + " 0.016206905211255562,\n", + " 0.019286710911544963,\n", + " 0.016659304559877103,\n", + " 0.018481970706874074,\n", + " 0.014478293336704112,\n", + " 0.015763380753411382]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pt_kuhn_result = pt_main(game, \n", + " env_configs,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVfr48c8zk94IKUBIKKHX0JuAYgcVVERFQQRUrKu7lt/id11F17p2115QV1TsiAorUhRQpCkgVVqAUEMgvc7M+f1xL5hgygCZFOZ5v17zysy9d+59zgzkySn3HDHGoJRSSh3hqO0AlFJK1S2aGJRSSpWhiUEppVQZmhiUUkqVoYlBKaVUGZoYlFJKlaGJQfk9EWkpIkZEAmo5DiMibWozBqVAE4M6hYhIqoicU+r1aBE5LCJn1HAcRkTyRCRXRHaLyDMi4qzJGJQ6GZoY1ClJRK4FXgIuNMb8UAshdDPGRABnA1cDN9TERWu71qNODZoY1ClHRG4EngbON8b8ZG87tjYxRUSmHfPWMSKyU0QOisg/jjn2YxH5r4jkiMg6EentTSzGmI3AIqCLfa4bRGSLiBwSkZki0rSCMgwSkV0iMsR+PVFENtg1oG9FpEWpY42I3Coim4HN3sSlVGU0MahTzc3AQ8DZxpgVx/neQUB7rL/y7xeRjqX2jQCmA9HATOBFb04oIp2AwcCvInIW8BhwBZAA7LDPeex7hgIfApcZY74XkYuB/wNGAvFYiebDY952CdAP6ORVSZWqhCYGdao5F/gZ+O0E3vugMabAGLMaWA10K7VvsTFmljHGDbx3zL7y/CIih4GvgDeBt4ExwFRjzC/GmCLgXmCAiLQs9b7LgdeAYcaYZfa2m4DHjDEbjDEu4FGge+lag73/kDGm4ATKrVQZmhjUqeZmoB3wpojIcb53X6nn+UBEJftCqmjP72mMaWiMaW2Muc8Y4wGaYtUSADDG5AIZQGKp9/0V+NgYs7bUthbA8yKSKSKZwCFAjnnfrqqLp5R3NDGoU81+rKagwcDLpbbnAWGlXjepyaBse7B+yQMgIuFALLC71DGXA5eIyB2ltu0CbjTGRJd6hB7pP7HpNMmq2mhiUKccY8werOQwVESetTevAkaLSKDdcTyqFkL7EJggIt1FJBirSWipMSa11DFHYr9DRG62t70K3CsinQFEpIGIXF6DcSs/o0Pb1CnJGLPT7uxdKCKFwD+xfjEfBn4APgBiajimuSLyT+AzoCHwEzC6nON2isjZwPciUmKMeVNEIoDpdr9CFvAd8EkNhq/8iOhCPUoppUrTpiSllFJlaGJQSilVhiYGpZRSZWhiUEopVUa9G5UUFxdnWrZsWdthKKVUvbJy5cqDxph4b46td4mhZcuWrFhxvFPgKKWUfxORHVUfZdGmJKWUUmVoYlBKKVWGJgallFJl1Ls+BqVU/VdSUkJaWhqFhYW1HcopJyQkhKSkJAIDA0/4HJoYlFI1Li0tjcjISFq2bMnxz46uKmKMISMjg7S0NJKTk0/4PNqUpJSqcYWFhcTGxmpSqGYiQmxs7EnXxDQxKKVqhSYF36iOz9V/EsOOJTD/YXCX1HYkSilVp/lPYkhbBgufBFdRbUeilKplmZmZvPzyy1Uf6CNXXXUVKSkpPPvss4wfP55PP/201mIpj/8kBnFaP427duNQStW62koMLpeLffv2sXz5ctasWcPf/va3Go/BG/6TGBx2YvBoYlDK302ePJmtW7fSvXt37rnnHp588kn69OlDSkoKDzzwAACpqal07NiRG264gc6dO3PeeedRUFAAwAsvvECnTp1ISUlh9GhrEb5Dhw5xySWXkJKSQv/+/VmzZg0AU6ZM4ZprrmHgwIFcc801nHfeeezevZvu3buzaNGiMnHNmzePHj160LVrVyZOnEhRURHLly9n5MiRAHz55ZeEhoZSXFxMYWEhrVq18snn4z/DVY/WGDy1G4dSqowHv1rH+j3Z1XrOTk2jeGB45wr3P/7446xdu5ZVq1YxZ84cPv30U5YtW4YxhhEjRrBw4UKaN2/O5s2b+fDDD3njjTe44oor+Oyzzxg7diyPP/4427dvJzg4mMzMTAAeeOABevTowYwZM5g/fz7jxo1j1apVAKxfv57FixcTGhpKamoqF1100dF9b731FmCN1Bo/fjzz5s2jXbt2jBs3jldeeYXbbrvt6LGLFi2iS5cuLF++HJfLRb9+/ar1czvCj2oMdlG1xqCUKmXOnDnMmTOHHj160LNnTzZu3MjmzZsBSE5Opnv37gD06tWL1NRUAFJSUhgzZgzTpk0jIMD6+3rx4sVcc801AJx11llkZGSQnW0lvBEjRhAaGlppHJs2bSI5OZl27doBcO2117Jw4UICAgJo3bo1GzZsYNmyZdx5550sXLiQRYsWMXjw4Gr/PMAvawyaGJSqSyr7y74mGGO49957ufHGG8tsT01NJTg4+Ohrp9N5tCnpm2++YeHChXz11Vc88sgj/Pbbb5VeIzw8/KRiPP3005k9ezaBgYGcc845jB8/HrfbzZNPPnlS562IH9UYtI9BKWWJjIwkJycHgPPPP5+pU6eSm5sLwO7duzlw4ECF7/V4POzatYszzzyTJ554gqysLHJzcxk8eDDvv/8+AN9//z1xcXFERUV5HVP79u1JTU1ly5YtALz33nucccYZAAwePJjnnnuOAQMGEB8fT0ZGBps2baJLly4nVP6qaI1BKeV3YmNjGThwIF26dGHYsGFcffXVDBgwAICIiAimTZuG0+ks971ut5uxY8eSlZWFMYbbb7+d6OhopkyZwsSJE0lJSSEsLIx33333uGIKCQnh7bff5vLLL8flctGnTx9uuukmAPr168f+/fs5/fTTAaspa9++fT67SVCMMT45sa/07t3bnNBCPaunwxc3wl9+gdjW1R+YUsprGzZsoGPHjrUdximrvM9XRFYaY3p7837/aUrSUUlKKeUV/0kMOipJKaW84j+JQfsYlFLKK/6TGHRUklJKecWniUFEhorIJhHZIiKTKznuMhExIuJVx8iJBaM1BqWU8obPEoOIOIGXgGFAJ+AqEelUznGRwB3AUl/FApSqMWjns1JKVcaXNYa+wBZjzDZjTDEwHbi4nOP+BTwB+HbxV60xKKVO0KOPPlpt5xoyZAgnNOS+BvkyMSQCu0q9TrO3HSUiPYFmxphvKjuRiEwSkRUisiI9Pf3EotFRSUqpE3S8icEYg6cet07UWueziDiAZ4C7qjrWGPO6Maa3MaZ3fHz8CV5QawxKKUtqaiodOnRgzJgxdOzYkVGjRjFr1iwuueSSo8d89913XHrppUyePJmCggK6d+/OmDFjAHjmmWfo0qULXbp04bnnnjt6zvbt2zNu3Di6dOnCrl27eOKJJ+jatSvdunVj8uQ/ulk/+eQT+vbtS7t27f409XZd4MspMXYDzUq9TrK3HREJdAG+t2/rbgLMFJERxpjqr2fpqCSl6qbZk2Ff5ZPQHbcmXWHY45UesmnTJt566y0GDhzIxIkTWbduHRs3biQ9PZ34+HjefvttJk6cyPDhw3nxxRePTn29cuVK3n77bZYuXYoxhn79+nHGGWfQsGFDNm/ezLvvvkv//v2ZPXs2X375JUuXLiUsLIxDhw4dvbbL5WLZsmXMmjWLBx98kLlz51Zv+U+SL2sMy4G2IpIsIkHAaGDmkZ3GmCxjTJwxpqUxpiXwM+CbpABaY1BKldGsWTMGDhwIwNixY/nxxx+55pprmDZtGpmZmSxZsoRhw4b96X2LFy/m0ksvJTw8nIiICEaOHHn0r/4WLVrQv39/AObOncuECRMICwsDICYm5ug5jiy8U3oq77rEZzUGY4xLRG4DvgWcwFRjzDoReQhYYYyZWfkZqpmOSlKqbqriL3tfOXYCOhFhwoQJDB8+nJCQEC6//PKjay14y9vptY9M5+10OnG5XMd1jZrg0z4GY8wsY0w7Y0xrY8wj9rb7y0sKxpghPqstgNYYlFJl7Ny5kyVLlgDwwQcfMGjQIJo2bUrTpk15+OGHmTBhwtFjAwMDKSkpAawpsGfMmEF+fj55eXl88cUX5S6Yc+655/L222+Tn58PUKYpqa7zozufdVSSUuoP7du356WXXqJjx44cPnyYm2++GYAxY8bQrFmzMrOTTpo06eiqbT179mT8+PH07duXfv36cf3119OjR48/nX/o0KGMGDGC3r170717d5566qkaK9vJ8p9pt/eugdcGw5XToOPw6g9MKeW12p52+8i6y2vXrv3Tvttuu40ePXpw3XXX1UJk1UOn3fbSil3W2qsldbA9TylVN/Tq1Ys1a9YwduzY2g6lVvnNCm7bMgroDbhdLgJrOxilVK1q2bJlubWFlStX1kI0dY/f1BgcDisHetxaY1CqLqhvzdj1RXV8rn6TGI6s3+rWxKBUrQsJCSEjI0OTQzUzxpCRkUFISMhJncdvmpLETgxGRyUpVeuSkpJIS0vjhOc+UxUKCQkhKSnppM7hN4nB6TzSlKSJQanaFhgYSHJycm2HoSrgN01JYvcxaFOSUkpVzm8SQ0CA1ZTk8WhiUEqpyvhNYnDYTUlGm5KUUqpSfpMYjoxK8mjns1JKVcpvEoPDYd3WpjUGpZSqnN8kBqf2MSillFf8JzFoH4NSSnnFbxKDQ/sYlFLKK36TGAIC7D4GTQxKKVUpv0kMzqNTYmgfg1JKVcZvEkOA04HbiNYYlFKqCn6TGJwOwY1DO5+VUqoKfpMYAp0OPDi0xqCUUlXwm8RwtMagfQxKKVUpv0kMAXZiwGiNQSmlKuM3icHpEKspye2p7VCUUqpO85vEEOh0WE1JWmNQSqlK+U1iOFJjQPsYlFKqUn6TGI72MXi0KUkppSrjP4nhSFOSDldVSqlK+U9icAgeo6OSlFKqKn6TGJxHm5I0MSilVGX8JjHofQxKKeUdv0kMIoJHtMaglFJV8ZvEAFjDVY2OSlJKqcr4VWIwOBBtSlJKqUr5VWLwiPYxKKVUVfwrMeDUG9yUUqoK/pUYRJuSlFKqKn6VGIwOV1VKqSr5NDGIyFAR2SQiW0Rkcjn7bxKR30RklYgsFpFOvozHiAPRUUlKKVUpnyUGEXECLwHDgE7AVeX84v/AGNPVGNMd+DfwjK/iAfCIU5uSlFKqCr6sMfQFthhjthljioHpwMWlDzDGZJd6GQ4YH8ZjD1fVGoNSSlUmwIfnTgR2lXqdBvQ79iARuRW4EwgCzirvRCIyCZgE0Lx58xMOyIgTMUUn/H6llPIHtd75bIx5yRjTGvg7cF8Fx7xujOltjOkdHx9/4tfSPgallKqSLxPDbqBZqddJ9raKTAcu8WE8do1B+xiUUqoyvkwMy4G2IpIsIkHAaGBm6QNEpG2plxcCm30Yj1VjQGsMSilVGZ/1MRhjXCJyG/At4ASmGmPWichDwApjzEzgNhE5BygBDgPX+ioe0BqDUkp5w5edzxhjZgGzjtl2f6nnd/jy+n8iDhzax6CUUpXyqilJRIaLSK13VJ8sI05tSlJKqSp4+8v+SmCziPxbRDr4MiBfMuLEoU1JSilVKa8SgzFmLNAD2Aq8IyJLRGSSiET6NLrqpp3PSilVJa+bh+y7lD/FGlaaAFwK/CIif/FRbNXP4dQ+BqWUqoK3fQwXi8gXwPdAINDXGDMM6Abc5bvwqpcRJw6tMSilVKW8HZU0EnjWGLOw9EZjTL6IXFf9YfmIjkpSSqkqeduUtO/YpCAiTwAYY+ZVe1S+4nDiQDuflVKqMt4mhnPL2TasOgOpEdqUpJRSVaq0KUlEbgZuAVqLyJpSuyKBH30ZmE84NDEopVRVqupj+ACYDTwGlF6BLccYc8hnUfmK1hiUUqpKVSUGY4xJtddMKENEYupdcnA4cWrns1JKVcqbGsNFwEqs1dWk1D4DtPJRXL7hcGiNQSmlqlBpYjDGXGT/TK6ZcHxLxEmAaGJQSqnKVNX53LOy/caYX6o3HN8Sh1Vc43EjDmctR6OUUnVTVU1JT1eyz1DBGs11lsManet2uQgI0sSglFLlqaop6cyaCqQmHKkluNwuAgiu5WiUUqpuqqop6SxjzHwRGVnefmPM574JyzeOJAa3y1XLkSilVN1VVVPSGcB8YHg5+wxQrxJDcGAgABm5BYRHNqjlaJRSqm6qqinpAfvnhJoJx7fiGkQAsHnPYZonNKnlaJRSqm7ydtrtWBF5QUR+EZGVIvK8iMT6Orjq1ig2BoDte9NrORKllKq7vJ1EbzqQDlwGjLKff+SroHwlKMxacC5t/4FajkQppeoub9djSDDG/KvU64dF5EpfBORTQVZT0t4DGbUciFJK1V3e1hjmiMhoEXHYjyuAb30ZmE8EhQOQnZNFXpGOTFJKqfJUNVw1hz/mSPorMM3e5QBygbt9Gl11sxNDOIVkFpQQHuxthUkppfxHVaOSImsqkBphNyWFUUhuodYYlFKqPF7/ySwiDYG2QMiRbccu91nnHakxSCG52pSklFLl8ioxiMj1wB1AErAK6A8sob7NlWQnhjCKtI9BKaUq4G3n8x1AH2CHPX9SDyDTZ1H5SuCRxFCoiUEppSrgbWIoNMYUAohIsDFmI9Ded2H5iDMAjzOEcCnSpiSllKqAt30MaSISDcwAvhORw8AO34XlQ0HhhBVpjUEppSriVWIwxlxqP50iIguABsBsn0XlQxIUTrgUsq/YXduhKKVUneTtXEnvHXlujPnBGDMTmOqzqHxIgiOIkCJydLiqUkqVy9s+hs6lX4iIE+hV/eHUgKBwIh06KkkppSpSaWIQkXvtu59TRCTbfuQAB4AvayTC6hYUToQmBqWUqlClicEY85h99/OTxpgo+xFpjIk1xtxbQzFWr6AIItAb3JRSqiJVzZXUwR6a+omI9Dx2vzHmF59F5itB4YRJIXnFmhiUUqo8VY1KuhOYBDxdzj5DfbvzGSAonFBTSG6RjkpSSqnyVDWJ3iT755k1E04NCAonxBSSW1hS25EopVSd5O1w1UARuV1EPrUft4lIoBfvGyoim0Rki4hMLmf/nSKyXkTWiMg8EWlxIoU4LkERBJtCCgqLfX4ppZSqj7wdrvoK1vDUl+1HL3tbhewhrS8Bw4BOwFUi0umYw34FehtjUoBPgX97H/oJsifScxfn+/xSSilVH3k7JUYfY0y3Uq/ni8jqKt7TF9hijNkGICLTgYuB9UcOMMYsKHX8z8BYL+M5cfaaDI7iHIwxiIjPL6mUUvWJtzUGt4i0PvJCRFoBVfXeJgK7Sr1Os7dV5DoqmGZDRCaJyAoRWZGenu5lyBUIaQBABPkUlGgHtFJKHcvbxHAPsEBEvheRH4D5wF3VFYSIjAV6A0+Wt98Y87oxprcxpnd8fPzJXSw0GoAG5PLK91sxxpzc+ZRS6hTj7SR680SkLX9Mtb3JGFNUxdt2A81KvU6yt5UhIucA/wDO8OKcJ8+uMQxtE8rD87fQq0VDhrRv5PPLKqVUfVHVDW4jK9jVRkQwxnxeyduXA21FJBkrIYwGrj7m/D2A14ChxpgD3od9EkKsGsP4HtFMPRDCf+Zv4Yx28drXoJRStqpqDMMr2WeAChODMcYlIrcB3wJOYKoxZp2IPASssGdofRKIwLqzGmCnMWbE8RTguNmJIaAkh4mD+vPwNxtIO1xAs5gwn15WKaXqi6pucJtwMic3xswCZh2z7f5Sz885mfOfELspiYJM2jaNBOBATqEmBqWUsnl7g1usiLwgIr+IyEoReV5EYn0dnE84A6whq4VZxIYHAXAwV292U0qpI7wdlTQdSAcuA0bZzz/yVVA+FxINhZnERQQDkKGJQSmljvL2BrcEY8y/Sr1+WESu9EVANSI0GgqziLFrDBm5vh8MpZRS9YW3NYY5IjJaRBz24wqsTuX6KaQBFGQSFOAgKiSAjDytMSil1BHeJoYbgA+AIvsxHbhRRHJEJNtXwflMiFVjAIiLCOag1hiUUuoob29wi/R1IDUqpAEUZgIQGxGkfQxKKVWKt6OSrjvmtVNEHvBNSDUg9I8aQ2y41hiUUqo0b5uSzhaRWSKSICJdsGZCrb+1iJAGUJQNHjdxkUHax6CUUqV425R0tT0K6TcgD7jaGPOjTyPzJfvuZ+tehmAO5xfjcnsIcHqbJ5VS6tTlbVNSW+AO4DNgB3CNiNTfW4Uj7EnzcvYSFxGEMXA4X5f6VEop8L4p6Svgn8aYG4EzgM1Yk+TVT9HNrZ+Zu2gUFQLA3qyCWgxIKaXqDm8TQ19jzDwAY3kauNR3YfnY0cSwk9bx1lKf29LzajEgpZSqOypNDCLy/wCMMdkicvkxu8f7KiifC4+HgBDI2knzmHCcDmFrem5tR6WUUnVCVTWG0aWe33vMvqHVHEvNEYEGzSBzJ0EBDprHhGmNQSmlbFUlBqngeXmv65foZpBpLUndKi5cawxKKWWrKjGYCp6X97p+adAMsuzEEB/O9oN5uD31u0hKKVUdqrqPoZs9F5IAoaXmRRIgxKeR+Vp0c8hLh+J8WsdHUOTysCdTV3JTSqlKawzGGKcxJsoYE2mMCbCfH3kdWFNB+kRMK+tnxmZaxUcAVNicdDivmP/74jfe/SkVl9tTUxEqpVSt8HY9hlNP0+7Wzz2raN2+AwBb0/MY0r7sYS63h9un/8riLQcxBopdHm44vVUNB6uUUjXHf+eAaJhszZm051diwoNoEBrItmNqDG6P4a5PVrNo80GeGJlCn5YNee/nHXi0L0IpdQrz38QgAgndYc+viAit40uNTNq/Hv7dms2vjWXuqq3cc357rujTjHEDWrLzUD4/bE6v3diVUsqH/DcxgNWctH8duIpoFR/xx70M676A/IN02P81LzeeyS1DWgNwfucmxIQH8emKtFoMWimlfMvPE0MP8JTAgfW0jo/gQE4ROYUluDd/x1pHBz5yXMjpWTORNGtaqKAAByO6NeW79fvJ0kn3lFKnKP9ODAlHOqB/pUOCtbzEZ4tWI3tX8V1xV1pc/igSmQBf3QFuKxGM6pVEsdvDBS8s4s6PVv2pX0Ippeo7/04MDVtaazPsWcXgNnG0igtnyYKvcWAYdslV9O/YEi54Eg6sh1/eBaBLYgPemdCHto0jmLN+PxPeWc5hXehHKXUK8e/EIGL1M+z5lQCng7vOa08Hxy4MQoeUAdYxHS6EpL6w+DlwWQlgSPtGvDOhL+9O7MuezAIe+no9FOfB+pmw6kPI2VeLhVJKqZPj34kBrH6GAxugpJALUxK4rXMxEpMMQfYd0CJwxt+t6TN+er7MW3u1aMjNZ7Rm/aollDzdBT6+BmbcBP/pBRu/qYXCKKXUydPEkNTX6oDesRiAwIMboFGnsse0ORu6XAYLHoM9q8rsurV7ANOCnyDX5YBrv4IbF0JcW/h8EhzcXFOlUEqpaqOJoc3Z1o1uqz+CkkI4tBUadSx7jAhc+AyERsPcB6xt+Ydg3kMEv3cR4U43VxX8nYPx/SChG1z5PgQEw8fjoDi/5suklFInQRNDQDB0Hgkbv4Y9v4Dx/LnGAFZSGHwXbPseFj0NL/WFxc9CTCvSL/mAjZ5Epv28wzq2QSKMfMNqovrh8RotjlJKnSxNDABdL4eSfJj3L+v1kXmUjtX7OkjqA/MeAnHCpB9gwje06DqYC7sm8PL3W9l+0L5Jrs3Z0PlSWPmO1TGtlFL1hCYGgGb9IDQGdv4E8R3/mHn1WIEhMOZT6H8LjPsSElKO7npgeCdCAhzc9N5Kcgrtm9/63QiFWbDqgxoohFJKVQ9NDADOAGh3vvW84/DKjw2NhqGPQaMOZTY3igrhlbG92JKey1PfbrI2NutnPRY+CUU5PghcKaWqnyaGI7qMAkeA1fxzgga2iePyXkl8uHwXy1MPUew2cP5jkLvf6pdQSql6QBPDEW3PgXu2QuNyOp6Pwy1D2uD2GC5/dQmPztoASb0gZTQseQkOba+mYJVSync0MZQWGn3Sp2geG8bnN5/GOR0b89HyXdZ0Gec8AI5AmDvFq3N4PIY3F23jpQVbMEbXflBK1SxNDD7QrVk095zfnoISN7dP/5U9noYw4BZYPwP2rq7wfTsz8tlyIJe7PlnNw99s4MlvN/HmIq1lKKVqliYGH2nfJJL7LuzIyh2HuefT1TDgNmvCvvkPA9bqcKVrA8YYJryzjPOfW8gXv+7m9rPacGHXBB6dvYFZv+2trWIopfyQT9d8FpGhwPOAE3jTGPP4MftPB54DUoDRxphPfRlPTbt+sDXs9eFvNrB0r5t+A++AeQ/iTl3ClbMNLWLDefqKbqzalcmatEy2pueRHBdOm0YR3HFOO0rcHvZmFXDbB7/QMSEKhwhvXtubxlEhtVwypdSpTHzVhi0iTuB34FwgDVgOXGWMWV/qmJZAFHA3MNObxNC7d2+zYsUKX4TsEwXFboY8tQBjYPqEFFp9MIhsE8agjHvJJoJ/XdyZR2dtpKDETWigk+X3nUNE8B/5OrfIxQNfrmPjvmxSD+YRHRbE86O707tlTC2WSilV34jISmNMb2+O9WVTUl9gizFmmzGmGJgOXFz6AGNMqjFmDeDxYRy1KjTIyX8n9sNjDOOnreOdpvcTkruTl6PeJTE6lH9+uY6wICfjBrTg7vPbl0kKABHBATx9RTe+uX0w0ycNwOkQxr61lC0H9L4IpZRv+DIxJAK7Sr1Os7cdNxGZJCIrRGRFenp6tQRXk9o3ieT1cb3Zl1XIv9bGsKDReAYV/8jXFzt44aoefHHLQB66uAvXDUqu9Dxdkxrw6U0DCAsK4C8frqKwxF1DJVBK+ZN60flsjHndGNPbGNM7Pj6+tsM5IT2bN+SLW09j/l1ncP4ND0NUIg3n3s2I9hE0jw3z+jyNokJ46vIUNuzN5on/bcQYQ36xy4eRK6X8jS8Tw26gWanXSfY2v9W5aQNaxIZbiwBd+hoc2gZf3grH2c9zVofGjD+tJW//mMplr/xEv0fm8doPW5nw9jKmzFxHscv7lrndmQVMeHsZuw75bnpwYwz/+no9z8zZ5LNrKKWqjy9HJS0H2opIMlZCGA1c7cPr1S/Jg+HcB2HOfdbjrH9ak/R5afKwDvy8LYNfdmYSGRLAY7M3khMdt+MAABvRSURBVBgdyoJN6USFBDCie1NW7crisp6JiEiZ9+46lE/T6FDmbtjPtJ93sGjzQZ757neKXG7+dk472jaOJO1wPk/P+Z2okACCAhyc26kJfVs0IP+TGwnqfBEBXS7xKs4Ne7OZuXoPby227sdo0ziS8zs3JjjA6f1npZSqUT4blQQgIhdgDUd1AlONMY+IyEPACmPMTBHpA3wBNAQKgX3GmM6VnbO+jUqqlDHw1e3wy3+h9Vkw9nNrUSAv7c8uZN2eLFrHR7B4y0Gu6N2Mez5ZzYxVe44e89Tl3RjVK+no689/SePOj1czoFUsS7ZlANAgNJCsAmtG2L7JMTx0cWfGvLGU/GKrD8Pl8VDiNtzbNZcbN08CILdJfwLjW7Gj4400adWZZdsOkRwfTuvfnofGnaHzJfy+P4eLX/yRghI3/ZJjOJBTxPaDeXRvFs1nN5+G0+F9WZVSJ+d4RiX5NDH4wimVGI748Xn47n4Y+xm0OeekTlVY4mbm6j1k5Bbz7bp97DqUz/f3DCEyJJDM/GKufO1nNu23RjQNaR/PpNNbkVvoYtJ7K+mSGMXa3dkEOITYiCDev74/bRpFUFDk4v03nyRs7zKuDpjPStMBjJsOjl2kexrwj0Yvsn3PAc5JcvPQgdshKpFVly3k5g9WU+I2TLu+L20bRZJTWMIHy3by7/9t4tFLu3J1v+bV8ekppbygiaG+cRXBi33A4YQbFlTLnE0Aa9IyGfHij4zskciGfTls2JsNwK1ntmbrgTzuu6gjSQ3DMMbww+/pDGgdy6cr0/h9Xw7jTmtJ6/gI60Sb58L7lwGwP7QN/Q4/RGJ0KH2cm3g67//Y6kmghRzAiZsAsfo37g24m0WBA3l1bC+6JDY4GpMxhlGvLuFATiEL7zmTmV99RmFWOldec3O1lFkpVT5NDPXRjp/g3eHQpCtc9Cw07eHd+/attWZuLcyEPtdbK8eVMv7tZXy/KZ3E6FCu7tec8CAnY/u3IMDp5biDnH3w4VXWWtiFWZgBtzE1/HoGtomlQ5MoCtZ+TdGnN3IgOJmowjT2Jg6lQ/aP7M8uIrxJG+Iz14CnBLpfbZUL+HjFLu7/dDlPXZBI/3mjCKYE152badgg8ng+MaXUcdDEUF+tmwGz/x943HD7L7BrGWTvgZ7jyvY9ZGyFrfPBVQgLHrMWGgoMg5y9MGE2NEyG32dDz/H8np7H83M3c+8FHUhq6P2wWACWvQHf/h+4i+GSVyC+PcS1h+CIMofl5eURHBzCxHeWsWl/LmMS93HL9ttxBgQgPcfBwc2QuhjuWAVRieQvfYeC/z1ArPxxk97ivq8y6IKrrNpTUQ6Ex53MJ6mUOoYmhvpsz6/w+pnQchDsXAIeF5x+D5z2F1j+JuTst37pZ+60jm8xEC57C4Ij4ZUB1mJDUYmQughGvgEpV5xYHPvXW+drcy6c/yjEt6vyLQs2HmDCO8sBuK/DPq4/r6dV88lKg+dSrNpQYRYc3s7W0K4sKmzN2b070XD5sywLHczspL9yU+odJAdk4PjLSgizpv1wuT3e13CUUuXSxFDfzX/Yah5q3AXi2sGqaRAUCcU54AwCBEZ/ALGtyq5PveMneO9SqyYRGAZhcXDLkj/9he+VhU9acdy1CSKbePUWj8cw+o2fiYsI4unLuxMaVGpI6oLHYN0XEJUAPa/F1eFiHA4HDoew4dWxdNz3FXkmmGBKEIEFkcPJOfMRVqdl8936/cy984yj5zPG/GkIrlKqcpoYTgVHvhdjYM4/rHUczrPumKYw02rWKc+On2DD19DuPCtJtBsGI/4Dm+dAhwshJApcxRAQVPn13zjL+nnD/OorU0WK88lb/DKOzFS2NLmQzKUfMDhrJt+5e/GPkok8HDiV/hEH2HPVd2w65OGRbzbw4tU96ZusEwkq5S1NDMry86vwv79btYeSfAiPh26jYflbcNWH0GpI+e/L2QdPt4ez7rOasWqa24X58Tlk/r/Id0YS4C4kiBIedI1jbtSl7DpUQFCAg/93fntG9kwiJryKJKeUqjOzq6ra1v8muOApqylo+PMQFA4//cdKEjNvh6Jc67jCbKvje9NsyNoNv39rbW9/Qe3E7QxABt8FrYYQ5snj577/YY2zE9c7Z5Fx6DA3D2nNaa1jefibDfR9ZC4PfrUOl/uUnaBXqRqnNQZ/kpUGv75vdQh/eCW0PR+imsKaj6DYThIIRDS2+jL+uua47sSudkU5cHgHNOkCqT/ieecivvP0puft04lr2IDfUvfywarDTF++i1vPbM3d57U/2vewLT2Xuz9ZzYtX96RpdGjtlUGpOkKbklTVFj8Lc6dAQAh0Hgk9r7FGNM2dAjt+hL43wgX/ru0oy3AvfgHn3H9CUIQVd8Eh6DKKeXuDmZR2Hg3CQ+nVoiGZ+cWEBwfw/aZ0Lu7elPkbDhAc6OC5K3swqK0Og1X+SRODqpoxsG8NxLa1Zns9Ims3fD4Jhj4GCSm1F19Fdi2HNdOt2oQzENZ9CcU5/NTzKT7K68WBHevZWBDN4aI/ajrhQU6aNAjhYG4xX/9lEM1ijvN+DqVOAZoYlP/weOA/Pa3mr66jYNbdlARFc13Q4yQ2b8uHv+xnwsCWjD+tJRe+sJguiVG8f31/ncBP+Z3jSQy+nHZbKd9zOKDfjfC/ybB7JST2JnDvav4b9zpm0zr6J4zm9ITTaRjVmofOT+LOmamc/u8FXD84meiwQM7r1ITwYP1voFRpWmNQ9Z/bBd8/Cms/gzGfwYJHYN3nVj+Eq9A6JrwR5B1gXbf7eGDfQFbsOAxAv+QYzIENxLTozH3Dux7/tCFK1RPalKT8W9pKmH41XPWBNe9UVhoseRGMB/auwQy8g40txzJ7u5vfFnzM20FP8pYZwWuB1/Cvoc2YtjqbcQNacm6nxpVeJjO/GBGhQWhgDRVMqROniUGp8hRmWwsjrZsBAcGYVkMo2b6EoJIsPAEhZLuCiCabOe5ePBV0C99EP8lnch7BA29k16EC9mcX4nQI0WFBnNWhETdPW0l6ThE3D2nNXedVcCe6UnWEJgalKpP+Oyx91ZqhNibZGpr7ybW4m6SwztmJLjuncdATQSPJJMNEst7TgmTHPtIkgalczCD3Ula42/GVGcgZbWL5fnMGD47ozJV9mvHmom0czC3m/y7oSFCA3j+q6g5NDEodr4LDENzA6sz+6UWY8w8OBzaiYckBighiU8yZdHWvR7LSAMMhZxwf9v+am3fdxc/7hTFZN9ExIZrf9+fg8hguSkngxat7Vn7JYjeH8otJ1BvwVA3QUUlKHa/Qhn88738LeFw0bDcUts4jKKEbKS0HQe4BeOdCCAghZt8abs19AXb+yGnAmpgdzE5vxRQmMqpPK6Yv38X1gzOJyd9G2u7d/OxqR4vYcC6z19/OK3Ix+vWf2X4wjyX3nkVkSNl+CmMM2QUuGoRp/4WqeVpjUOp4eDzWGhnPdoK8dIjvCMmnY/avhR0/kRPcmPCYpizb6+KF4uE8EvAWiZLB0OLH2W4SGNI+HqcIaYcLjq69/fjIrozuW3b96+fm/s5/5m/huSu7M7xb09ooqTrFaFOSUr6W/jvs+QWa9f1jTYwtc2Hp61CcR0H6NkLy9yIYjDggLI45gWfxz8wLSAx1kSExPHhxZ576aiWNJYtePXuzIyOPkT2T2Jqey4Mz1+NwQAtXKn+NXUbnyybTPOsXMttcQlpWcZl1tJXyhiYGpWpbXga8McR6ftFz1nDZrfMxIdEIBi58BnYtJXftbELydvMP13W4HUEM4ydKCGBtcHfGXHYZETOuJbJoP/kmmDAp4q8ltzLDPZBXx/bk/M5NMAYcDqHY5WHehv2c2aERwQGOPy1k5PYYvdvbz2liUKouKMi01rCObGw1QX18DaStsBZachVaN+BFJVISEkPgHmtJ1Lzw5gQ5hcDsHdY5AsMpaHUuoZtmkCkNcAdHM00u4r38AbR0bycswHDrNVcz9acdfLtuP50Soth5KJ+r+jbjznPbk1fs4v4v1/Ld+v3cO6wjEwcl1+IHomqTJgal6iKPB4zbukN763xrrYyQKHCXwLbvrXUyOlwEDqe19veeX63X4fGQux+2L4LPrwcgU6KINtkAbDFNSfPEsy1xBP9KbU+HhGg27M0mLiIIt8dQUOImOS6CLQdyaBgWREpSA5IahtGzRUOGdm7CjFW76dMyhuS48Fr8cJSvaWJQ6lSVs8/qy5j/CAy8g/WHDDlL36VjWC5RBbtwxXXCmTKK4mVTOVgcxOeNbuPKuO2EJXUle/aDzIi/kakHO5NX5MJTUsA1IT+yoKgdkU3b0ygqlD7Jcdxwequq41D1jiYGpfxIkctNsANY9wUseBQObYUmXa2mLPu+i6Oa9YfrvsWzfyOF711BWO4O8hwRBLiLeNk1gufdl/HA8E6MP63ln/opVP2miUEpf+V2wc4l1mip9I3w3kjodxMUZVk38f06DToOh83fQXAknP8oZtHTSPpGcoKb8FtAF77IbEVebAoPh39MTJdzoN1Qft1XzNS1LrolNeD6wVqjqI80MSilLB6PdTc3WM1Qz3ezOr27XAaD/gbRzax9K6bC138DoCgwmqISF1Hk4nYE4XEGsbKoOePc9+PyeJh526Cjw2XTDueTmV+iw2frAU0MSqny5eyz7vIOCP7z9qfbW/sKDmMCQrgv9B/cn/0gweLCjYOc/nfz8rIsZgUP5Z0h+STmbeCmJQ35ISeBjglRXB6xmhS28H3cVRz2hDGqVxJ7MguZu2E/NwxuRXJcOKFBztopt9LEoJQ6AT+9CE17wO//gyYpeLqMYutPn+PKSKXjrw8dPewdx0hGuWcRIYWUGCdrEi5jQ2EMozKnEkIxu00c15oHyCgOJNJRzE6Ptc52gEM4s0MjRvVKYsavu4mPDObvQzv8aaEkt8ewcV82nRKitJ+jGmliUEpVH4/HmgLEeCCxN2z6hhJHKK+3/g+XF31Go11zrGG4kQkUD3uGwJk3QUkBHrcbwUN2ykTmJd5K2r69ZK34mAx3OD+FnUVefh7nNsmjVae+zF67l2YxYcSGB5GZX8L/1u3j/os6MXFQMpn5xTzxv02s25PFpNNbcVFK2SlCfkvLom3jCEICnbBvrXUnelAlCy4ZAyvegrj2kDy4yuIbY47eSFjaOz9uZ9WuTB4bmUJokJMffk9n/ob93HpmG7YfzKNJgxA+W5nG2R0bExMeVPFa466iP9fggPScIuIigqotOWpiUEpVr7QV1i+vxl2sGkVwJLQcZO1zuyB7N4THQVA4HNgAqz6w7scoyoXlb0CTFCjKhsOpABR3uATXzhWE5afxvOtSVja6jPSSUMKzt7KiKInmMWHsOpxPz+YNyStysS09j6SGoWw7mMfr1/SiSYMQFm0+SGigk+nffIsjthWJ+Rt5y/NPSOoLYz+FEKvfw+1245g7BWnS2epbmfkXWP0hOIMw7Ydx+PclEBBMTFwTaNQRc8ZkHl6URUGJm9vObMNt76/kYHYBf+sbSuuIYgKb9+bDZTv57xLrJsTW0Q46RhYyd18IhSUeggMcFLk8JEaHUpC5nzMdq/jcM4gR3RKJ8mTTu3NbRnRrytLth5j5/otM8bzEP51/5fxLx9Hk8Eq+3JhHs7bdOPzdv2k/7FbOax8Dsa0pdHkQgeCAE2uO08SglKo71s2AuVOg4BBc9RGs/gB++S807UFJeAKBm2dhxIFEJkD2bg52HEtEix7MToVpB9uQtj+dx0e0Y3DwZvbPuI8Fnh7086zm7yU3ECwlfBj0CJtpRjAlRJJPFHnkhDUndNQr/LBxDxt//h+3Oz4GwBMah6PgIEuajqe9ayMhuWksyGlKMCV0jnUQl7kaJ26We9qTa0IpIYA2jr0EOw3B7jwiKGBI0TPkSBgT+jamQ6tkoubcxYC8edzY4BU6tGrJyrXryQptxu8H8vih2Ru0SF/A9/FjKd6/kXMcK3nPdQ4j4vfxU14ig4sWEiX55JkQMgknUTIA2OWJp5kjnSyJpIHJ4dPoiTx7eCD3XNyPS3o2O6GvQRODUqpuMcZqMgkMsZ4XZlod3cZYd3iv+wJ2/wIR8dbzI5xB4C4++rI4KJqg4kwA8hv15HBWFo0C8ggIiaCkqJDrD40BZxDPOJ4nTrKPvm+loytzTR+6mQ3MLu7B1wzG7bF+97WMDSOv2E16ThH9ojMZLos5w/0zkQEeHMU5hIaFE1CcjXEVgcdFbkQy4UX7cRo3XPCkNZrLVQgxra1O/JI8XA1bsz7xClLWPnZ0vXEjTohtjRz8nZ2mEfFkUhjdht29J9N2x4fsOpTPvhYjaFP4G002vMPaiNNok7OcbCKIkAJwBHBo8EMknXndCX0FmhiUUvVXVhogVpPUlu+gQZLVRBUcBR0uYu+mZTTOWYfj28nWcaPfhw4XArBhbzZJDUPZuGMP2xf8lyaJzRjcNp5tIV248K31JEaH8vKYXrSKD2f22n3sPlzABV2b4HQIGbnFdEiILNtUYwyIQOYuKM6FDV/BqvehaU84tA32rrKO6z7GamLrcJF1c+GCR6z7RhK6w9jPYcdiSOoDQeFsW/IFf1nVnLED2zC6T7M/9yEYA3tXkx/biadmrWN0GzftvrzIujflgich/sSWkdXEoJQ6tbmK4eeXoPXZkJDi1VvSDufTMCzoT6OgTjyGIvj1PasfZdBfy+7btxY2fg2n/cVKaierOB8CQ60kdYI0MSillCrjeBKDrlaulFKqDJ8mBhEZKiKbRGSLiEwuZ3+wiHxk718qIi19GY9SSqmq+SwxiIgTeAkYBnQCrhKRTsccdh1w2BjTBngWeMJX8SillPKOL2sMfYEtxphtxphiYDpw8THHXAy8az//FDhb9B54pZSqVb5MDInArlKv0+xt5R5jjHEBWUDssScSkUkiskJEVqSnp/soXKWUUlBPOp+NMa8bY3obY3rHx8fXdjhKKXVK82Vi2A2Uvnc7yd5W7jEiEgA0ADJ8GJNSSqkq+DIxLAfaikiyiAQBo4GZxxwzE7jWfj4KmG/q240VSil1ivHpDW4icgHwHOAEphpjHhGRh4AVxpiZIhICvAf0AA4Bo40x26o4Zzqw4wRDigMOnuB76xotS92kZambtCzQwhjjVVt8vbvz+WSIyApv7/yr67QsdZOWpW7SshyfetH5rJRSquZoYlBKKVWGvyWG12s7gGqkZambtCx1k5blOPhVH4NSSqmq+VuNQSmlVBU0MSillCrDbxJDVVOA1xYRSRWR30RklYissLfFiMh3IrLZ/tnQ3i4i8oJdhjUi0rPUea61j98sIteW2t7LPv8W+73VNkmhiEwVkQMisrbUNp/HXtE1fFCWKSKy2/5uVtn35RzZd68d1yYROb/U9nL/ndk3ei61t39k3/Tpk6nnRaSZiCwQkfUisk5E7rC317vvppKy1LvvRkRCRGSZiKy2y/LgiV6/uspYIWPMKf/AusFuK9AKCAJWA51qOy47tlQg7pht/wYm288nA0/Yzy8AZgMC9AeW2ttjgG32z4b284b2vmX2sWK/d1g1xn460BNYW5OxV3QNH5RlCnB3Ocd2sv8NBQPJ9r8tZ2X/zoCPsW7gBHgVuNl+fgvwqv18NPBRNZQlAehpP48EfrdjrnffTSVlqXffjf1ZRdjPA4Gl9md4XNevzjJWGGt1/ZKoyw9gAPBtqdf3AvfWdlx2LKn8OTFsAhLs5wnAJvv5a8BVxx4HXAW8Vmr7a/a2BGBjqe1ljqum+FtS9pepz2Ov6Bo+KMsUyv/lU+bfD/Ct/W+s3H9n9i+Eg0DAsf8ej7zXfh5gHyfV/B19CZxbn7+bcspSr78bIAz4Beh3vNevzjJW9PCXpiRvpgCvLQaYIyIrRWSSva2xMWav/Xwf0Nh+XlE5KtueVs52X6qJ2Cu6hi/cZjevTC3VLHK8ZYkFMo01tXzp7WXOZSqZev5E2c0PPbD+Oq3X380xZYF6+N2IiFNEVgEHgO+w/sI/3utXZxnL5S+JoS4bZIzpibXS3a0icnrpncZK8fVyTHFNxO7ja7wCtAa6A3uBp310HZ8QkQjgM+Cvxpjs0vvq23dTTlnq5XdjjHEbY7pjzTbdF+hQyyGVy18SgzdTgNcKY8xu++cB4Ausfyz7RSQBwP55wD68onJUtj2pnO2+VBOxV3SNamWM2W//R/YAb2B9N1QRc3nbM4BosaaWP7YsPpl6XkQCsX6Rvm+M+dzeXC+/m/LKUp+/Gzv+TGABVrPO8V6/OstYLn9JDN5MAV7jRCRcRCKPPAfOA9ZSdjrya7HaVbG3j7NHkfQHsuxq+7fAeSLS0K5Sn4fVhrgXyBaR/vaokXGlzuUrNRF7RdeoVkd+wdkuxfpujlx/tD1qJBloi9UZW+6/M/sv5wVYU8sfG3O1Tz1vf15vARuMMc+U2lXvvpuKylIfvxsRiReRaPt5KFZfyYYTuH51lrF81dkxVJcfWCMvfsdq0/tHbcdjx9QKa+TAamDdkbiw2gTnAZuBuUCMvV2Al+wy/Ab0LnWuicAW+zGh1PbeWP9ptgIvUo0dm8CHWNX4Eqx2y+tqIvaKruGDsrxnx7rG/s+YUOr4f9hxbaLUSK+K/p3Z3/Uyu4yfAMH29hD79RZ7f6tqKMsgrCacNcAq+3FBffxuKilLvftugBTgVzvmtcD9J3r96ipjRQ+dEkMppVQZ/tKUpJRSykuaGJRSSpWhiUEppVQZmhiUUkqVoYlBKaVUGZoYlF8SEbf8MTPnKqlixl0RuUlExlXDdVNFJO5kz6OUL+lwVeWXRCTXGBNRC9dNxbpP4GBNX1spb2mNQalS7L/o/y3WWgPLRKSNvX2KiNxtP79drPUB1ojIdHtbjIjMsLf9LCIp9vZYEZkj1vz7b2LdTHbkWmPta6wSkdfsCdacIvKOiKy1Y/hbLXwMys9pYlD+KvSYpqQrS+3LMsZ0xbqj97ly3jsZ6GGMSQFusrc9CPxqb/s/4L/29geAxcaYzlhzYTUHEJGOwJXAQGNNquYGxmBNCpdojOlix/B2NZZZKa8EVH2IUqekAvsXcnk+LPXz2XL2rwHeF5EZwAx72yDgMgBjzHy7phCFtQDQSHv7NyJy2D7+bKAXsNyaDohQrAnnvgJaich/gG+AOSdeRKVOjNYYlPozU8HzIy7EmluoJ9Yv9hP5A0uAd40x3e1He2PMFGPMYaAb8D1WbeTNEzi3UidFE4NSf3ZlqZ9LSu8QEQfQzBizAPg71lTIEcAirKYgRGQIcNBY6wYsBK62tw/DWiITrInmRolII3tfjIi0sEcsOYwxnwH3YSUfpWqUNiUpfxUq1kpaR/zPGHNkyGpDEVkDFGEtW1maE5gmIg2w/up/wRiTKSJTgKn2+/L5Y7rkB4EPRWQd8BOwE8AYs15E7sNavc+BNavrrUAB8La9DaylGZWqUTpcValSdDipUtqUpJRS6hhaY1BKKVWG1hiUUkqVoYlBKaVUGZoYlFJKlaGJQSmlVBmaGJRSSpXx/wFaPnD8vVffkgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "x = [i*1000 for i in range(len(tf_kuhn_result))]\n", + "\n", + "plt.plot(x, tf_kuhn_result, label='tensorflow')\n", + "plt.plot(x, pt_kuhn_result, label='pytorch')\n", + "plt.title('Kuhn Poker')\n", + "plt.xlabel('Episodes')\n", + "plt.ylabel('Exploitability')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "game = \"leduc_poker\"\n", + "num_players = 2\n", + "env_configs = {\"players\": num_players}\n", + "num_train_episodes = int(3e6)\n", + "eval_every = 100000\n", + "hidden_layers_sizes = [128]\n", + "replay_buffer_capacity = int(2e5)\n", + "reservoir_buffer_capacity = int(2e6)\n", + "anticipatory_param = 0.1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Losses: [(0.5390237, 14.245871), (0.44778812, 15.658735)]\n", + "[100000] Exploitability AVG 1.6898823214088354\n", + "_____________________________________________\n", + "Losses: [(0.46877638, 13.020107), (0.46317512, 14.113714)]\n", + "[200000] Exploitability AVG 1.420086811832837\n", + "_____________________________________________\n", + "Losses: [(0.5191153, 12.782812), (0.44340974, 9.09498)]\n", + "[300000] Exploitability AVG 1.181871429941547\n", + "_____________________________________________\n", + "Losses: [(0.42321426, 9.537439), (0.33284122, 9.16)]\n", + "[400000] Exploitability AVG 1.0245515925064417\n", + "_____________________________________________\n", + "Losses: [(0.4591559, 7.893066), (0.5145581, 13.236103)]\n", + "[500000] Exploitability AVG 0.8958223060182396\n", + "_____________________________________________\n", + "Losses: [(0.4845383, 5.753558), (0.38578534, 12.685564)]\n", + "[600000] Exploitability AVG 0.8159920007228245\n", + "_____________________________________________\n", + "Losses: [(0.37469828, 6.724424), (0.39382324, 15.418528)]\n", + "[700000] Exploitability AVG 0.7346057720733029\n", + "_____________________________________________\n", + "Losses: [(0.49615866, 5.774213), (0.3718758, 10.98672)]\n", + "[800000] Exploitability AVG 0.7057984875903107\n", + "_____________________________________________\n", + "Losses: [(0.48984215, 9.576526), (0.3671657, 12.342072)]\n", + "[900000] Exploitability AVG 0.6972515890040387\n", + "_____________________________________________\n", + "Losses: [(0.41772717, 8.931775), (0.3413425, 10.916821)]\n", + "[1000000] Exploitability AVG 0.6737432047433465\n", + "_____________________________________________\n", + "Losses: [(0.41556114, 8.729887), (0.31489578, 10.115262)]\n", + "[1100000] Exploitability AVG 0.6852156897082571\n", + "_____________________________________________\n", + "Losses: [(0.48483667, 7.0489373), (0.3854279, 6.4418135)]\n", + "[1200000] Exploitability AVG 0.6621826836650042\n", + "_____________________________________________\n", + "Losses: [(0.41613883, 4.942773), (0.444838, 9.591253)]\n", + "[1300000] Exploitability AVG 0.6766518776527517\n", + "_____________________________________________\n", + "Losses: [(0.44554454, 10.306429), (0.37035146, 8.447489)]\n", + "[1400000] Exploitability AVG 0.5827228998283466\n", + "_____________________________________________\n", + "Losses: [(0.44904405, 7.180885), (0.47336185, 7.2617435)]\n", + "[1500000] Exploitability AVG 0.5291790984510443\n", + "_____________________________________________\n", + "Losses: [(0.5205521, 8.924693), (0.48343286, 6.811921)]\n", + "[1600000] Exploitability AVG 0.5022623112379254\n", + "_____________________________________________\n", + "Losses: [(0.4813504, 5.84527), (0.51469016, 10.681292)]\n", + "[1700000] Exploitability AVG 0.48630643816425995\n", + "_____________________________________________\n", + "Losses: [(0.50044, 4.661733), (0.36980015, 8.9714365)]\n", + "[1800000] Exploitability AVG 0.4935168184226987\n", + "_____________________________________________\n", + "Losses: [(0.49431032, 9.019564), (0.41029614, 8.568829)]\n", + "[1900000] Exploitability AVG 0.47340736864293803\n", + "_____________________________________________\n", + "Losses: [(0.43172938, 5.960622), (0.407834, 10.395099)]\n", + "[2000000] Exploitability AVG 0.4510645650692823\n", + "_____________________________________________\n", + "Losses: [(0.483876, 6.110518), (0.44114056, 9.112255)]\n", + "[2100000] Exploitability AVG 0.4561857999571657\n", + "_____________________________________________\n", + "Losses: [(0.48100927, 5.3415275), (0.45580634, 9.493247)]\n", + "[2200000] Exploitability AVG 0.4317089958795024\n", + "_____________________________________________\n", + "Losses: [(0.5539425, 3.4461184), (0.41182294, 13.08461)]\n", + "[2300000] Exploitability AVG 0.4208609647837446\n", + "_____________________________________________\n", + "Losses: [(0.4976226, 10.506472), (0.54511666, 7.152174)]\n", + "[2400000] Exploitability AVG 0.421839634060961\n", + "_____________________________________________\n", + "Losses: [(0.42836452, 9.636693), (0.41147757, 10.273931)]\n", + "[2500000] Exploitability AVG 0.40056422894510424\n", + "_____________________________________________\n", + "Losses: [(0.5118806, 7.83749), (0.4875685, 12.062527)]\n", + "[2600000] Exploitability AVG 0.411465908371901\n", + "_____________________________________________\n", + "Losses: [(0.48534644, 7.544059), (0.45399478, 8.825887)]\n", + "[2700000] Exploitability AVG 0.38803892703328446\n", + "_____________________________________________\n", + "Losses: [(0.45197463, 9.399324), (0.43891603, 13.119152)]\n", + "[2800000] Exploitability AVG 0.367132770241162\n", + "_____________________________________________\n", + "Losses: [(0.507201, 7.622658), (0.37780696, 8.293583)]\n", + "[2900000] Exploitability AVG 0.3633606374587205\n", + "_____________________________________________\n", + "Losses: [(0.46548223, 3.9901786), (0.46427155, 10.7747755)]\n", + "[3000000] Exploitability AVG 0.35713582160929336\n", + "_____________________________________________\n" + ] + }, + { + "data": { + "text/plain": [ + "[1.6898823214088354,\n", + " 1.420086811832837,\n", + " 1.181871429941547,\n", + " 1.0245515925064417,\n", + " 0.8958223060182396,\n", + " 0.8159920007228245,\n", + " 0.7346057720733029,\n", + " 0.7057984875903107,\n", + " 0.6972515890040387,\n", + " 0.6737432047433465,\n", + " 0.6852156897082571,\n", + " 0.6621826836650042,\n", + " 0.6766518776527517,\n", + " 0.5827228998283466,\n", + " 0.5291790984510443,\n", + " 0.5022623112379254,\n", + " 0.48630643816425995,\n", + " 0.4935168184226987,\n", + " 0.47340736864293803,\n", + " 0.4510645650692823,\n", + " 0.4561857999571657,\n", + " 0.4317089958795024,\n", + " 0.4208609647837446,\n", + " 0.421839634060961,\n", + " 0.40056422894510424,\n", + " 0.411465908371901,\n", + " 0.38803892703328446,\n", + " 0.367132770241162,\n", + " 0.3633606374587205,\n", + " 0.35713582160929336]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf_leduc_result = tf_main(game, \n", + " env_configs,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Losses: [(tensor(0.7167), tensor(10.2624)), (tensor(0.6678), tensor(24.5570))]\n", + "[100000] Exploitability AVG 1.7760776446223332\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6730), tensor(14.0845)), (tensor(0.6229), tensor(16.1690))]\n", + "[200000] Exploitability AVG 1.4667133317281158\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6050), tensor(15.1587)), (tensor(0.5255), tensor(16.4437))]\n", + "[300000] Exploitability AVG 1.2494437745597449\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5169), tensor(9.0873)), (tensor(0.4858), tensor(12.4749))]\n", + "[400000] Exploitability AVG 1.120871150332515\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6774), tensor(8.6445)), (tensor(0.4650), tensor(11.7802))]\n", + "[500000] Exploitability AVG 1.1015036650424368\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4969), tensor(7.8441)), (tensor(0.6199), tensor(11.1916))]\n", + "[600000] Exploitability AVG 1.0245672591492947\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5393), tensor(11.3621)), (tensor(0.4865), tensor(9.8605))]\n", + "[700000] Exploitability AVG 0.9445766513183707\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5734), tensor(8.2119)), (tensor(0.6477), tensor(13.5363))]\n", + "[800000] Exploitability AVG 0.8684752481367197\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5339), tensor(7.7509)), (tensor(0.5705), tensor(11.3290))]\n", + "[900000] Exploitability AVG 0.8006560040555245\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5618), tensor(6.6120)), (tensor(0.4335), tensor(8.4792))]\n", + "[1000000] Exploitability AVG 0.8018183519995761\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6756), tensor(6.5384)), (tensor(0.5442), tensor(9.3602))]\n", + "[1100000] Exploitability AVG 0.7926676079915461\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4992), tensor(7.3925)), (tensor(0.6278), tensor(10.9842))]\n", + "[1200000] Exploitability AVG 0.7992893814920394\n", + "_____________________________________________\n", + "Losses: [(tensor(0.7034), tensor(9.9266)), (tensor(0.5977), tensor(8.8878))]\n", + "[1300000] Exploitability AVG 0.8035728366445121\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6024), tensor(12.8139)), (tensor(0.5742), tensor(8.4938))]\n", + "[1400000] Exploitability AVG 0.756405194780331\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5650), tensor(8.5757)), (tensor(0.6259), tensor(9.5099))]\n", + "[1500000] Exploitability AVG 0.742653918177091\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6178), tensor(8.7429)), (tensor(0.7075), tensor(11.4955))]\n", + "[1600000] Exploitability AVG 0.6744911938268123\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5109), tensor(7.4196)), (tensor(0.6720), tensor(13.6984))]\n", + "[1700000] Exploitability AVG 0.6481299471024332\n", + "_____________________________________________\n", + "Losses: [(tensor(0.4995), tensor(6.0341)), (tensor(0.6266), tensor(8.3702))]\n", + "[1800000] Exploitability AVG 0.6340046779741395\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5942), tensor(8.8162)), (tensor(0.5066), tensor(13.7095))]\n", + "[1900000] Exploitability AVG 0.6052965706103838\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5576), tensor(8.4911)), (tensor(0.6388), tensor(10.2203))]\n", + "[2000000] Exploitability AVG 0.6071257458704584\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5786), tensor(6.6841)), (tensor(0.6548), tensor(10.6600))]\n", + "[2100000] Exploitability AVG 0.5775614842977357\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5866), tensor(6.8267)), (tensor(0.5127), tensor(9.2956))]\n", + "[2200000] Exploitability AVG 0.5430772967943114\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5920), tensor(8.0974)), (tensor(0.6018), tensor(11.9147))]\n", + "[2300000] Exploitability AVG 0.5485013972001643\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6663), tensor(8.1932)), (tensor(0.5554), tensor(9.4191))]\n", + "[2400000] Exploitability AVG 0.5387629731446001\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6010), tensor(7.6481)), (tensor(0.5952), tensor(5.8818))]\n", + "[2500000] Exploitability AVG 0.5407604559108162\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6029), tensor(6.8392)), (tensor(0.6507), tensor(11.3971))]\n", + "[2600000] Exploitability AVG 0.5217140821286138\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5471), tensor(7.0737)), (tensor(0.6330), tensor(10.9512))]\n", + "[2700000] Exploitability AVG 0.4710156288266728\n", + "_____________________________________________\n", + "Losses: [(tensor(0.8002), tensor(7.7415)), (tensor(0.4751), tensor(8.6963))]\n", + "[2800000] Exploitability AVG 0.4674406844154157\n", + "_____________________________________________\n", + "Losses: [(tensor(0.6338), tensor(6.9319)), (tensor(0.5234), tensor(9.9743))]\n", + "[2900000] Exploitability AVG 0.45240516373248907\n", + "_____________________________________________\n", + "Losses: [(tensor(0.5559), tensor(9.8301)), (tensor(0.4386), tensor(11.8644))]\n", + "[3000000] Exploitability AVG 0.4589802299847634\n", + "_____________________________________________\n" + ] + }, + { + "data": { + "text/plain": [ + "[1.7760776446223332,\n", + " 1.4667133317281158,\n", + " 1.2494437745597449,\n", + " 1.120871150332515,\n", + " 1.1015036650424368,\n", + " 1.0245672591492947,\n", + " 0.9445766513183707,\n", + " 0.8684752481367197,\n", + " 0.8006560040555245,\n", + " 0.8018183519995761,\n", + " 0.7926676079915461,\n", + " 0.7992893814920394,\n", + " 0.8035728366445121,\n", + " 0.756405194780331,\n", + " 0.742653918177091,\n", + " 0.6744911938268123,\n", + " 0.6481299471024332,\n", + " 0.6340046779741395,\n", + " 0.6052965706103838,\n", + " 0.6071257458704584,\n", + " 0.5775614842977357,\n", + " 0.5430772967943114,\n", + " 0.5485013972001643,\n", + " 0.5387629731446001,\n", + " 0.5407604559108162,\n", + " 0.5217140821286138,\n", + " 0.4710156288266728,\n", + " 0.4674406844154157,\n", + " 0.45240516373248907,\n", + " 0.4589802299847634]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pt_leduc_result = pt_main(game, \n", + " env_configs,\n", + " num_train_episodes,\n", + " eval_every,\n", + " hidden_layers_sizes,\n", + " replay_buffer_capacity,\n", + " reservoir_buffer_capacity,\n", + " anticipatory_param)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV9f348df7Zi8IZLEChD3CkrAEAVEE/bpQ3KA4itaq3VU7fmprW63ftrZfR6t1VaxasW4piIqgIAKK7E2AMLIgJGSP9++Pc8ALZpHcm5vxfj4e53HmPZ/3yQ15c87ncz4fUVWMMcaY0+UJdADGGGNaJksgxhhjGsQSiDHGmAaxBGKMMaZBLIEYY4xpEEsgxhhjGsQSiDGAiKSLyLmBjgNARO4XkXmBjsOYulgCMS1ac/rDfyo3EZSLyDERyROR5SIyLtBxGeMrlkCM8a9XVTUaSAA+Bf4jItIUBYtIcFOUY9ouSyCmVRIRj4jcIyI7RSRXRP4tIh299s8WkT3uvl+c8tnnReRBr/XJIpLhtZ4sIv8RkWz384/VFY+qlgMvAJ2AOBHpIiJvi8hhEdkhIt+p4TpCRORlEXldRELdz73ulr1bRO7yOvZ+EZkvIvNEJB+Ycxo/MmNOmyUQ01rdCVwKTAK6AEeAxwFEZBDwJDDb3RcHdKvPSUUkCHgX2AP0BLoCr9Tjc2E4f9D3qWqO+5kMt/yZwO9EZMopn4kA3gRKgSuBCuAd4Gu33HOAH4jINK+PXQLMB2KBl+pzTcY0lCUQ01rdBvxCVTNUtRS4H5jpPtaZCbyrqkvdfb8Cqup53tE4f/R/qqqFqlqiqp/WcvyVIpIH7ANGAjNEJBkYD9ztfn4t8A/geq/PtQP+C+wEblTVSmAUkKCqv1bVMlXdBTwNXO31uRWq+qaqVqlqcT2vyZgGsWekprXqAbwhIt6JoRJIwkkA+45vVNVCEcmt53mTgT2qWlHP4/+tqrO8N4jIGOCwqhZ4bd4DpHmtjwVCgGv0mx5PewBd3IR0XBCwzGt9H8Y0EUsgprXaB9ykqp+dukNEDgIDvdYjcR5jHVcIRHqtdzrlvN1FJPg0ksipDgAdRSTGK4l0B/Z7HbMIWAd8KCKTVTXTLXu3qvat5dzWvbZpMvYIy7QGISIS7jUFA38DfisiPQBEJEFELnGPnw9cKCITRCQU+DUn/1tYC1wgIh1FpBPwA699XwAHgYdEJMotb/zpBKuq+4DlwO/dzw8FbgbmnXLcH4B/4SSReLfsAhG5W0QiRCRIRFJFZNTplG+Mr1gCMa3B+0Cx13Q/8BfgbWCRiBQAnwNjAFR1I/A9nD/OB3Eq2DO8zvciTkV1Os6dwKvHd7h1ERcBfYC97ueuakDM1+BUwh8A3gDuU9XFpx6kqr/BqUhfDLQHLgSGA7uBHJy6k/YNKN+YRhMbUMoYY0xD2B2IMcaYBrEEYowxpkEsgRhjjGkQSyDGGGMapMW9BxIfH689e/YMdBjGGNOirFmzJkdVE3x5zhaXQHr27Mnq1asDHYYxxrQoIrLH1+e0R1jGGGMaxBKIMcaYBvFbAhGRZ0UkS0Q21LC/vYi8IyJfi8hGEbnRX7EYY4zxPX/WgTwPPAb8s4b93wM2qepFIpIAbBWRl1S1zI8xGWOagfLycjIyMigpKQl0KK1OeHg43bp1IyQkxO9l+S2BqOpSEelZ2yFAjDu8ZzRwGGfAHGNMK5eRkUFMTAw9e/akiUb4bRNUldzcXDIyMkhJSfF7eYGsA3kMp0vtA8B64PuqWu2gPiIyV0RWi8jq7OzspozRGOMHJSUlxMXFWfLwMREhLi6uye7sAplApuF0m90Fp3fRx0SkXXUHqupTqpqmqmkJCT5txmyMCRBLHv7RlD/XQCaQG4H/qGMHTvfUA/xWWuYmWPgLKLdRPo0xxhcCmUD2AucAiEgS0B/Y5bfSju6DFY9Bhr2EaExbl5eXxxNPPBGw8q+55hqGDh3Kn//8Z+bMmcP8+fMDFktj+LMZ78vACqC/iGSIyM0icpuI3OYe8hvgTBFZD3wI3K2qOf6Kh+QxgMCeb41waoxpYwKVQCoqKjh06BCrVq1i3bp1/PCHP2zyGHzJbwlEVa9R1c6qGqKq3VT1GVX9m6r+zd1/QFXPU9UhqpqqqvPqOmejRMRCp1RLIMYY7rnnHnbu3Mnw4cP56U9/yiOPPMKoUaMYOnQo9913HwDp6ekMHDiQ73znOwwePJjzzjuP4mLnEfhf//pXBg0axNChQ7n66qsBOHz4MJdeeilDhw5l7NixrFu3DoD777+f2bNnM378eGbPns15553H/v37GT58OMuWLTsprg8//JARI0YwZMgQbrrpJkpLS1m1ahWXXXYZAG+99RYRERGUlZVRUlJCr169mupHVq0W1xdWo/SYAGueh4oyCA4NdDTGGOCBdzay6UC+T885qEs77rtocI37H3roITZs2MDatWtZtGgR8+fP54svvkBVufjii1m6dCndu3dn+/btvPzyyzz99NNceeWVvP7668yaNYuHHnqI3bt3ExYWRl5eHgD33XcfI0aM4M033+Sjjz7i+uuvZ+3atQBs2rSJTz/9lIiICNLT07nwwgtP7HvmmWcAp2XanDlz+PDDD+nXrx/XX389Tz75JHfccceJY5ctW0ZqaiqrVq2ioqKCMWPG+PTndrraVlcmPc6EimI48FWgIzHGNBOLFi1i0aJFjBgxgjPOOIMtW7awfft2AFJSUhg+fDgAI0eOJD09HYChQ4dy3XXXMW/ePIKDnf+Hf/rpp8yePRuAKVOmkJubS36+kxgvvvhiIiIiao1j69atpKSk0K9fPwBuuOEGli5dSnBwML1792bz5s188cUX/OhHP2Lp0qUsW7aMs846y+c/j9PRxu5AznTmez6D7oHN3MYYR213Ck1BVbn33nu59dZbT9qenp5OWFjYifWgoKATj7Dee+89li5dyjvvvMNvf/tb1q9fX2sZUVFRjYpx4sSJLFiwgJCQEM4991zmzJlDZWUljzzySKPO21ht6w4kKh4SBsCe5YGOxBgTQDExMRQUFAAwbdo0nn32WY4dOwbA/v37ycrKqvGzVVVV7Nu3j7PPPpuHH36Yo0ePcuzYMc466yxeeuklAJYsWUJ8fDzt2lX7alu1+vfvT3p6Ojt27ADgxRdfZNKkSQCcddZZPProo4wbN46EhARyc3PZunUrqampDbp+X2lbdyDg3IWsew0qKyCo7V2+MQbi4uIYP348qampnH/++Vx77bWMGzcOgOjoaObNm0dQUFC1n62srGTWrFkcPXoUVeWuu+4iNjaW+++/n5tuuomhQ4cSGRnJCy+8cFoxhYeH89xzz3HFFVdQUVHBqFGjuO02p9HqmDFjyMzMZOLEiYDzCO3QoUMBfxlTVDWgAZyutLQ0bdSAUuvnw+s3w9wl0GWEr8IyxpyGzZs3M3DgwECH0WpV9/MVkTWqmubLctrWIyzwqgexx1jGGNMYbS+BtOsCHVIg3d4HMcaYxmgzCWR7ZgG/fW8TJeWV0GM87F0OVdV2/muMMaYe2kwC2Xu4iKeX7WZ1+hHoOR6Kj0D2lkCHZYwxLVabSSBje8URGuRh6fbsk98HMcYY0yBtJoFEhQWT1rMDn2zNhtge0K6rJRBjjGmENpNAACb1S2BrZgGH8kudepA9y6GFNWM2xjS93/3udz471+TJk2nUqwjNSJtKIBP7OaMZLt3mPsY6lgm5OwMclTGmuTvdBKKqVLWBRjptKoEM6BRDYkwYn2zPdu5AwB5jGdMGpaenM2DAAK677joGDhzIzJkzef/997n00ktPHPPBBx8wY8YM7rnnHoqLixk+fDjXXXcdAH/6059ITU0lNTWVRx999MQ5+/fvz/XXX09qair79u3j4YcfZsiQIQwbNox77rnnxLlfe+01Ro8eTb9+/b7VpXtL0qb68hARJvZL4INNmVReNZygqATnMdbIGwIdmjFt14J74FDtnRGetk5D4PyHaj1k69atPPPMM4wfP56bbrqJjRs3smXLFrKzs0lISOC5557jpptu4qKLLuKxxx470aX6mjVreO6551i5ciWqypgxY5g0aRIdOnRg+/btvPDCC4wdO5YFCxbw1ltvsXLlSiIjIzl8+PCJsisqKvjiiy94//33eeCBB1i8eLFvr7+JtKk7EHDqQY4Wl/P1/qPOYyy7AzGmTUpOTmb8eOdJxKxZs/jss8+YPXs28+bNIy8vjxUrVnD++ed/63OffvopM2bMICoqiujoaC677LITdxE9evRg7NixACxevJgbb7yRyMhIADp27HjiHMcHiPLuIr4lalN3IAAT+sQjAp9szeaMHuNh01uQtxdiuwc6NGPapjruFPzl1I4IRYQbb7yRiy66iPDwcK644ooTY33UV327bT/eTXxQUBAVFRWnVUZz0ubuQDpEhTKsW6z7PsjxehDrF8uYtmbv3r2sWLECgH/9619MmDCBLl260KVLFx588EFuvPHGE8eGhIRQXl4OOF2rv/nmmxQVFVFYWMgbb7xR7cBOU6dO5bnnnqOoqAjgpEdYrYXfEoiIPCsiWSKyoZZjJovIWhHZKCKf+CuWU03sl8DX+/LIi+kD4e0h/dOmKtoY00z079+fxx9/nIEDB3LkyBG++93vAnDdddeRnJx8Um+2c+fOPTEK4RlnnMGcOXMYPXo0Y8aM4ZZbbmHEiG/37D19+nQuvvhi0tLSGD58OP/7v//bZNfWVPzWnbuITASOAf9U1W+NeiIiscByYLqq7hWRRFWteRQXV6O7cwfW7DnC5U8u57FrR3Dhhh9Bzja468tGndMYU3+B7s79+LjkGzZ8+/+3d9xxByNGjODmm28OQGS+0eK7c1fVpUBt92zXAv9R1b3u8XUmD18Z1q097cKDnbfSe46Hwzuh4FBTFW+MaaZGjhzJunXrmDVrVqBDaRECWQfSD+ggIktEZI2IXF/TgSIyV0RWi8jq7OzsRhccHOThrL4JLN2ejXa3frGMaWt69uxZ7d3HmjVrWLp06UljoZuaBTKBBAMjgf8BpgG/EpF+1R2oqk+papqqpiUkJPik8In94snML2WrJwVCo60i3Zgm1tJGQ20pmvLnGsgEkgEsVNVCVc0BlgLDmqrwE92a7DgCyWMsgRjThMLDw8nNzbUk4mOqSm5uLuHh4U1SXiDfA3kLeExEgoFQYAzw56YqvHP7CPolRfPJtmzm9jsTPvoNFOZCVFxThWBMm9WtWzcyMjLwxSNpc7Lw8HC6devWJGX5LYGIyMvAZCBeRDKA+4AQAFX9m6puFpH/AuuAKuAfqlpjk19/mNQvgReW76Hk7LGEA+xdAQMvbMoQjGmTQkJCSElJCXQYppH8lkBU9Zp6HPMI8Ii/YqjLxH4JPL1sNytKenB2cLjzGMsSiDHG1EubexPd26ieHQkP8fDJznzoNgr22AuFxhhTX206gYSHBDG2VxyfHB8f5NB6KDka6LCMMaZFaNMJBJx6kN05hWR2SAOtgn1fBDokY4xpEdp8AjnenPfDwu7gCbZ+sYwxpp7afALpFR9Ftw4RfLzzGHQ5w94HMcaYemrzCeT4KIXLd+RQ2f1MOPAllBUFOixjjGn22nwCAacepLCsku0RQ6GqAjKsHsQYY+piCQQ4s3ccwR5hwdEeIB57jGWMMfVgCQSICQ/hjB4dWLyrGDoNtQRijDH1YAnENalfAhsP5FPUZQxkrIKK0kCHZIwxzZolENcktznv157BUFEC+22EQmOMqY0lENegzu2Iiwrl7bwezgYbYMoYY2plCcTl8TjNeRfuKkeTUmHH4kCHZIwxzZolEC8T+8VzuLCMzG7TnK7d8/YFOiRjjGm2LIF4OauvUw/ygWeCs2HjGwGMxhhjmjdLIF7io8NI7dqOt/eFO92abJgf6JCMMabZsgRyikn9Evhybx4lA2bAwa8hZ0egQzLGmGbJEsgpJvZNoLJKWRExERC7CzHGmBr4LYGIyLMikiUitY5zLiKjRKRCRGb6K5bTcUaPDkSHBbNonwd6jIf180E10GEZY0yz4887kOeB6bUdICJBwMPAIj/GcVpCgjxM7BfP4s1ZVKVeDrnbnZEKjTHGnMRvCURVlwKH6zjsTuB1IMtfcTTEtMGdyC4oZV27ic4gU/YYyxhjviVgdSAi0hWYATwZqBhqcvaAREKChPd3lkOvs2HDf6CqKtBhGWNMsxLISvRHgbtVtc6/zCIyV0RWi8jq7OxsvwfWLjyEcb3jWbjxEJp6ORzdZ2OEGGPMKQKZQNKAV0QkHZgJPCEil1Z3oKo+pappqpqWkJDQJMFNG5zEntwitnecBMHhsOH1JinXGGNaioAlEFVNUdWeqtoTmA/crqpvBiqeU00dlIQILNhWCP2mOW+lV1YEOixjjGk2/NmM92VgBdBfRDJE5GYRuU1EbvNXmb6UGBPOGd07sHDjIUi9HAqzIX1poMMyxphmI9hfJ1bVa07j2Dn+iqMxpg1O4nfvbyEjfgLdQmNg/evQe0qgwzLGmGbB3kSvxbTBnQD479ajMPBC2PyOjVRojDEuSyC16BEXxYBOMSzamAmpM6H0qI0TYowxLksgdThvcCdW7TlMTuJYiIxzujYxxhhjCaQu0wYnoQqLtx6GQZfA1gVQeizQYRljTMBZAqnDoM7t6NYhwm2NNRMqip0kYowxbZwlkDqICNMGd+KzHbkUJKVBu672UqExxmAJpF6mDe5EWWUVS7blwuAZTkV6UV39RBpjTOtmCaQeRvboQFxU6DcvFVaVO016jTGmDbMEUg9BHmHqoCSWbM2mNHEodOxlXbwbY9o8SyD1NG1wJ46VVrB852GnMn33Mig4FOiwjDEmYCyB1NOZfeKIDgv+5jEWChubTd+PxhjT5OqVQETkIhFp08kmLDiIyf0TWLw5k8r4/pCUao+xjDFtWn2TwlXAdhH5g4gM8GdAzdm0wZ3IOVbGl3uPOHchGavgSHqgwzLGmICoVwJR1VnACGAn8LyIrHBHCYzxa3TNzOT+CYQGeVi44fhjLOydEGNMm1Xvx1Kqmo8z8NMrQGec8cy/FJE7/RRbsxMTHsL4PnEs3HQIje0O3UY746UbY0wbVN86kEtE5A1gCRACjFbV84FhwI/9F17zM21wJ/YdLmbzwQIYMhMyN8DelYEOyxhjmlx970AuA/6sqkNU9RFVzQJQ1SLgZr9F1wydOygJj+C0xhp+HcR0hgU/g6qqQIdmjDFNqr4J5JCqnjSeq4g8DKCqH/o8qmYsPjqMtB4dnQQSFg1Tfw0H18LaeYEOzRhjmlR9E8jUarad78tAWpLzBiex5VABe3OLYMgVkDwGFj8AxXmBDs0YY5pMrQlERL4rIuuBASKyzmvaDayr47PPikiWiGyoYf917rnWi8hyERnW8MtoWseHul248RCIwPl/gKJc+OThAEdmjDFNp647kH8BFwFvufPj00i3aW9tngem17J/NzBJVYcAvwGeqk/AzUFyx0gGdm7nJBCALsNh5A2w8u+QtSWwwRljTBOpK4GoqqYD3wMKvCZEpGMdH1wK1NjnuaouV9Uj7urnQLd6xtwsTBucxJq9R8guKHU2TPmVUyey4GegGtjgjDGmCdTnDgRgDbDana/xWveVm4Eah/lzX1pcLSKrs7OzfVhsw00b3AlV+GBTprMhKh7O/gXs/gS2vBvY4IwxpgnUmkBU9UJ3nqKqvdz58amXLwIQkbNxEsjdtcTxlKqmqWpaQkKCL4pttAGdYujeMfKbx1gAaTdDwkBY+HMoLw5ccMYY0wTqqkQ/o7apsYWLyFDgH8Alqprb2PM1JWeo2ySW78zhaHG5szEoGM5/GPL2wvL/C2yAxhjjZ8F17P9jLfsUmNLQgkWkO/AfYLaqbmvoeQLpkuFdeXrZbl5auYfbJ/dxNvaaBIMugWV/gmHXQGxyYIM0xhg/qTWBqOrZDT2xiLwMTAbiRSQDuA+nGxRU9W/A/wPigCdEBKBCVdMaWl4gpHZtz8R+CTyzbDc3nplCRGiQs+O8B2HbIvjgV3DF8wGN0Rhj/KXWBCIiU1T1IxG5rLr9qlpjT4Kqek1t51bVW4Bb6hVlM/a9yb256qnPeXXVXuaMT3E2xnaHCT+EJb9z6kVSzgpskMYY4wd1tcKa5M4vqma60I9xtRhjesUxqmcH/r50F2UVXv1hjb8L2neHBXdDZUXgAjTGGD+pqxXWfe78xmqmm5omxObv9rP7cPBoCW9+tf+bjSERMO23kLUR1jwXuOCMMcZP6tude5yI/FVEvhSRNSLyFxGJ83dwLcXkfgmkdm3Hk5/spLLK6yXCgRdByiT46EEobFGNzIwxpk717UzxFSAbuByY6S6/6q+gWhoR4XuT+7A7p5D31h/03uE06y0tgI8fDFyAxhjjB/VNIJ1V9TequtudHgSS/BlYSzNtcCd6J0TxxMc7UO+uTBIHwui5sPo52LcqcAEaY4yP1TeBLBKRq0XE405XAgv9GVhL4/EIt0/uw5ZDBXy4OevknZPvgcg4eOZc+L+R8O4PYeMbUJgTmGCNMcYHRGvp+E9ECnBeGBQgCjjezMgDHFPVdn6P8BRpaWm6erUvu+HynfLKKs7+3yXER4fxxu1n4r7f4sjbC5veht1LYc9nUHbM2Z6UCj3PgpSJ0ONMiIgNTPDGmFZNRNb4+l27WhNIc9ScEwjAi5/v4VdvbuBft4zhzD7x1R9UWQ4H1jodL6Yvg72fQ0UJiAc6D4e+U2HCjyAkvGmDN8a0WgFNICLSAegLnPirduowt02huSeQkvJKJv7hY/okRvOv74yt34cqSiFjlXN3snsp7F0BfafBVfMgONS/ARtj2gR/JJD6NuO9BViKU+/xgDu/35eBtBbhIUF856xeLN+Zy5d7j9T9AYDgMOg5Ac7+Odz0X7jwz7B9Icy/0blbMcaYZqi+lejfB0YBe9z+sUYANgB4Da4d053YyBCe+HhHw06QdhOc/4gzrsjrt9ib7MaYZqm+CaREVUsARCRMVbcA/f0XVssWFRbMjWemsHhzFpsP5jfsJGPmwrTfwaY34Y1boarSt0EaY0wj1TeBZIhILPAm8IGIvAXs8V9YLd+cM3sSFRrEE0t2Nvwk474H5z4AG+bDm7dbEjHGNCt1jQcCgKrOcBfvF5GPgfbUMgStgfaRIcwa14Onl+7iR1P7kRIf1bATTfgBVJU73aF4guHi/wNPffO+Mcb4T30r0V88vqyqn6jq28CzfouqlbhlQi9Cgjz8rTF3IQATfwqT7oa18+C9H0ILa3ptjGmd6vtf2cHeKyISBIz0fTitS0JMGFeNSuY/X2VwIK+RY6RPvtd5N2TN8/D+Ty2JGGMCrq4x0e9130YfKiL57lQAZAFvNUmELdytk3qjCk8t3dW4E4nAOf8PzrwTVj0NC39uScQYE1B1jQfye1WNAR5R1XbuFKOqcap6bxPF2KJ1jY1gxoiuvLJqLznHSht3MhGY+hsY8134/An44P9ZEjHGBExddyAD3MXXROSMU6cmiK9VuG1yb0orqnh6WSPvQsBJItN/D6NugeV/hY9/2/hzGmNMA9TVCutHwFzgj9XsU2BKTR8UkWdxhr3NUtXUavYL8BfgAqAImKOqX9Yz7hald0I0M4Z35bnP0rludA+6x0U27oQizouGlWWw9BEIi4Hx3/dNsMYYU091PcKa687PrmaqMXm4ngem17L/fJy+tfriJKkn6x92y3P3+QMI9ggPvrfJNyf0eODCR2HwZc6jrNXWKM4Y07Tq24w3RETuEpH57nSHiITU9hm3o8XDtRxyCfBPdXwOxIpI5/qH3rIktQvnzil9WbQpk6Xbsn1zUk8QXPaU0/Hiuz+C9fN9c15jjKmH+jbjfRKn2e4T7jSSxt8xdAX2ea1nuNu+RUTmishqEVmdne2jP74BcNOEnvSMi+SBdzZSXllV9wfqIygErnzB6YzxP3Nhq73faYxpGvVNIKNU9QZV/cidbsTpXLFJqOpTqpqmqmkJCQlNVazPhQUH8asLB7Ezu5AXlqf77sQhEXDNy9B5GPz7Btj1ie/ObYwxNahvAqkUkd7HV0SkF9DYjpn2A8le693cba3alAGJTO6fwF8Wbye7oJHNer2FxcCs1yGuN7x8DWQ03zFTjDGtQ30TyE+Bj0VkiYh8AnwE/LiRZb8NXC+OscBRVT3YyHM2eyLCry4cRElFJY8s3OLbk0d2hNlvQHQizLscDm3w7fmNMcZLvRKIqn6I01rqLuBOoL+qflzbZ0TkZWAF0F9EMkTkZhG5TURucw95H9gF7ACeBm5v4DW0OL0TorlpfAqvrcng630+HlYlphNc/xaERMKLMyC3kf1wGWNMDWod0lZELqvtw6r6H59HVIfmPqRtfRWUlDPlj5/QrUMEr992Jh6P+LaA7G3w3HQnkdy4AGKT6/6MMabVCsSQthfVMl3oy0DampjwEO6ePoCv9ubxxld+qPpJ6Aez/gMlR+HFS+FYlu/LMMa0abXegTRHreUOBKCqSrnsyeXszyvmox9PIia81ldrGmbPCudRVlwfuOFtp57EGNPmBOIO5HjBcSLyVxH5UkTWiMhfRCTOl4G0RR6P8MDFg8kuKOWxjxo4fnpdeoyDq1+CnK1OIik+4p9yjDFtTn1bYb0CZAOXAzPd5Vf9FVRbMiw5livTuvHsZ7vZlX3MP4X0OQeuegmyNrlJxMcV98aYNqm+CaSzqv5GVXe704NAkj8Da0t+Om0A4cFB/OZdH/WTVZ1+58GVLzpNe+dd5tSNGGNMI9Q3gSwSkatFxONOVwIL/RlYW5IQE8b3z+3Lx1uz+WhLpv8K6j8drvwnHFznvCdSku+/sowxrV59E8h3gH8Bpe70CnCriBSIiP0V8oEbzuxJ74Qofv3OJkorGvuSfy0GXABXPA8HvoKXZkJpgf/KMsa0avV9kTBGVT2qGuJOHndbjKq283eQbUFIkIf7LhpMem4Rz36a7t/CBl4IM591ujt56Qoo9VPdizGmVatvK6ybT1kPEpH7/BNS2zWxXwJTByXxfx9tZ3umn+8MBl0CM5+BfV9YEjHGNEh9H2GdIyLvi0hnEUkFPgdi/BhXm3XfRaRzslcAAB+/SURBVIOICgtm1jMr2Ztb5N/CBs+Ay5+GfZ/Dv66CskL/lmeMaVXq+wjrWuAFYD3wHvADVf2JPwNrq7p1iGTezWMorajiumc+59DREv8WmHo5XPY07F3uJhE/Jy1jTKtR30dYfYHvA68De4DZItLIgb1NTfp3iuGFG0dz+FgZs55ZyeHCMv8WOGQmzPg77PkMXr4ayov9W54xplWo7yOsd4BfqeqtwCRgO7DKb1EZhiXH8sycUew7XMT1z64kv6TcvwUOvRIufRJ2L4V5M+09EWNMneqbQEa7XbrjjmH+R2CG/8IyAGN7xfG3WSPZcrCAm59fRXGZH5v3Agy72nmcte9zeO4CyG/1w7MYYxqh1gQiIj8DUNV8EbnilN1z/BWU+cbZAxL5y9UjWLPnCLfOW+Pfd0QAhl4B1/4bjqTDM+dBznb/lmeMabHqugO52mv53lP2TfdxLKYG/zO0Mw9dNpSl27L5wStrqais8m+Bfc6BOe9CeZGTRGx4XGNMNepKIFLDcnXrxo+uHJXMry4cxIINh7j79fVUVfm5G/4uI+DmRRDeHl64CLZZzzXGmJPVlUC0huXq1o2f3TwhhR9N7cfrX2bw63c34fexXOJ6O0kkvi+8fA18Nc+/5RljWpTgOvYPc/u6EiDCq98rAcL9Gpmp1p1T+lBQUs7Ty3YTHRbMT6b192+B0Ykw5z14dTa89T04lgkTfgRiN6DGtHW13oGoapCqtnP7vAp2l4+v1zl8nohMF5GtIrJDRO6pZn93EflYRL4SkXUickFjLqYtEBF+fsFArhmdzGMf7+Dvn+z0f6FhMU7F+pAr4MNfw4KfQZWfK/ONMc1eXXcgDSYiQcDjwFQgA1glIm+rqvegF78E/q2qT4rIIOB9oKe/YmotRIQHLx3CsdJKfr9gC+0jQrh6dHf/FhocCjOegugkWPGYM8b6jL9DSDO/EVV1Jk99W6wbY+rLbwkEGA3sUNVdACLyCnAJ4J1AFDjem2974IAf42lVgjzCH68YRn5xOT9/Yz3tI0I4f0hn/xbq8cC030JMJ1j0SziyG+L7Q1WFO1U6c608eb2qwqmMb58MscnQvrs7T3bO5QmqvVxVp9v5wuyTp+IjzvaSfGdemu+1fNSdF0BwOEy6G8beDkH+/JU3pm0Rf1XEishMYLqq3uKuzwbGqOodXsd0BhYBHYAo4FxVXVPNueYCcwG6d+8+cs+ePX6JuSUqLqtk1jMrWZ9xlGfnjGJC3/imKXjda/DJQ06S8AR7TUHu5LUuQVB8GPL2OXNvnhBo1wViuztTZJxzTGGOc5dzPFlU1NAnmCcEwts5j9nC2jmT93p4O2cUxu0LodMQuPAv0G2k/38+xjQzIrJGVdN8es4AJ5AfuTH8UUTGAc8Aqapa44sOaWlpunq1vZfg7WhROVc9tYK9h4t46ZYxjOjeIdAh1az0GBzNgKP7IG+vO/daLsp1kkhUgjNFJ0JUPEQluusJ3+yL6AjBYXVX6KvC5necupuCQzB6Lkz5pZNcjGkjWloCGQfcr6rT3PV7AVT1917HbMRJMvvc9V3AWFXNqum8lkCql5Vfwsy/rSC/pJx/3zqOfknW2/63lOTDRw/CF085j87O/wMMvMhalJk2wR8JxJ81i6uAviKSIiKhOG+1v33KMXuBcwBEZCBO0+BsP8bUaiW2C2fezWMIDfIw+5mV7Dts3bJ/S3g7uOAPcMuHzl3Nv2c777fk7Qt0ZMa0SH5LIKpaAdwBLAQ247S22igivxaRi93Dfgx8R0S+Bl4G5qjf345rvbrHRfLPm0dTXFbJ7GdWkl1QGuiQmqduI+E7S+C8B2H3J/D4GFjxOFRWBDoyY1oUvz3C8hd7hFW3NXuOMOsfK0mJj+KVW8fSLrzOV3barry98N5P3Er2oTDuDqeyPb4vBNnPzbQeLaoOxF8sgdTPJ9uyueWFVYxI7sALN40mIrSOprJtmSpsegsW3A3HDjnbgkIhYQAkpUKnVHc+BCI7BjZWYxrIEgiWQE7HO18f4K5XvuLs/on8ffZIQoLsZbpaVZY73ddnboBD65155kan+5bjYjo7ySR5DIy/y2kFZkwLYAkESyCn66WVe/jFGxu4eFgX7r94MB2jQgMdUstzLBsy1zvvk2RudBPLBkiZCFe9ZM2BTYtgCQRLIA3x+Mc7eGThVkKChKmDkrgiLZmJfRMI8ljz1Qb7+hWnc8nEgXDd6xCTFOiIjKmVJRAsgTTU5oP5vLY6gze+yuBIUTmd2oVz+ciuXDEymZ7xUYEOr2XavthpChyVALPfcLq/N6aZsgSCJZDGKquo4sPNmby2JoMlW7OoUhid0pEr05K5YEgnIkOtr6jTkrEaXroCxAOz5jsDcRnTDFkCwRKIL2Xml/D6lxm8tjqD3TmFRIUGceHQLozvG0+X9uF0jo0gKSaMYKt8r13OdnjxMqcPr6tehN5TAh2RMd9iCQRLIP6gqqzec4TXVu/j3XUHKSr7ZqwPj0BiTDidY8Pp0j6Czm5i6dI+nO5xkQzs1A6P1aVA/kGYdznkbIMZf4MhMwMdkTEnsQSCJRB/KymvZO/hIg7kFXPwaAkH84rZn1fCwaPO+oG8YkorvunrMjEmjKmDkjhvcCfG9YojNLgN360U58Er18Kez2D6wzD2tkBHZMwJlkCwBBJoqsqRonIO5BWzLbOAxZszWbI1m6KySmLCgpk8IJGpg5KY3D+hXm/AV1Uph/JL2JVdyK6cY+QVlTO+TxzDkzu0zFZi5SXw+s2w5V2Y8EM45z7rrNE0C5ZAsATSHJWUV7J8Zw6LNmayeHMmOcfKCAkSxvWO57xBSUwdlERUWDC73SSxM7uQXdnH2JVdyO6cQorLvz08blxUKFMGJHLuoCTO6hvfoMr90opKth06RkFJOWf2aaJxUsAZI+W9H8Oa52D4LLjoLzaQlQk4SyBYAmnuKquUr/YeYdGmTBZuPMSe3G/3CuwR6NYhkl4JUfSKjyYlIYre8VH0SogmIjSIpduyWbw5k4+2ZFFQUkFYsIcJfeI5d1AS5wxIJLHdt4fRPVZawaYD+Ww8cJSNB/LZeCCf7ZkFVFQ5v9+/mzGEa8f4edhfb6qw5CFn0K0OPZ3x5FNnQuKApovBGC+WQLAE0pKoKtuzjvHh5iyqVOmd4CSJ7h0jCQ+pu2+u8soqVu0+zAebM/lgUyYZR4oBGJ4cy7kDE/F4hI0H8tl0IJ/dOYUnPhcfHcqgLu1J7dKOwV3a8+rqfazYmcMrc8cyskcT92W16W1Y/azT669WQdIQp4I99XJnWF9jmoglECyBtFWqytbMAhZvyuSDzVl8vS8PgG4dIkjt0p7BXdoxuKuTMBJjwhCveoejReVc/PinFJVV8u6dE0iq5g7G7woyYeMbsGE+ZKxytnUf5ySSwTOc8UmM8SNLIFgCMY7cY6UEezy0j6xfl+tbDxUw44nP6N8phlfmjiUsOIC9Ex/eDRteh/WvQfYWZ8z43lNg8KXQdSTE9bU6E+NzlkCwBGIabsH6g3z3pS+5Ki2Zhy4fctJdSkCoOp0zbpgP61+Ho3ud7UFhkDTI6T6+01BnnjQYwmyYYtNwlkCwBGIa538XbuWxj3fwm0tTmT22R6DD+YaqczdyaD0cWufMD65z3m4/rmMvN6kMce5YupxhTYRNvVkCwRKIaZzKKuWWF1axbHsOL88dy6iezXiAKFUoOHhyUjm0Hg7vcvYnDoLh18HQqyA6IbCxmmbPEgiWQEzjHS0u59LHP6OgpIJ37hxP5/YRgQ7p9BQfgY1vwlfzYP9q8ARDv+kwYhb0mWr1J6Za/kggfu13QkSmi8hWEdkhIvfUcMyVIrJJRDaKyL/8GY8xAO0jQnhq9kiKyyq47cU1lFTzImOzFtEB0m6E73wIt6+Esd+FfSvh5avhTwNh0a8ge2ugozRtgN/uQEQkCNgGTAUygFXANaq6yeuYvsC/gSmqekREElU1q7bz2h2I8ZWFGw9x64truGJkN/4wc2jgK9Ubo7Ictn/g3JVs+y9oJXQbBQMvAk8IVJY5x1SWei2XnbzceTgMu8Yeh7VSLeoRloiMA+5X1Wnu+r0Aqvp7r2P+AGxT1X/U97yWQIwv/WnRVv760Q5+fclgrh/XM9Dh+MaxLFj3qpNMsrd8e39QGASFQnCoMw8KAQTy9jiPw/pfACNvgF5ngyeAzZ2NT/kjgfjzYWlXYJ/XegYw5pRj+gGIyGdAEE7C+e+pJxKRucBcgO7dm7A7CtPq/eDcfmw8kM+v39lE/6QYxvSKC3RIjRedCGfeCePugKLDThIIcpOFJ6jmlltZW+CrF2Htv2Dz29C+u1OvMmIWtO/atNdgWoRA970dDPQFJgPXAE+LSOypB6nqU6qapqppCQl2e218x+MR/nz1cLrHRXL7S1+yP6840CH5jghExUFELIRGOpXrtT2mSxwA034LP94CM59zhuhd8jt4NNUZdXHzu87jLmNc/kwg+wHvzn66udu8ZQBvq2q5qu7GqTPp68eYjPmWduEhPDU7jdKKKmY8/hnLtmcHOqTACg6D1Mvg+jfh+1/DWT92mg+/eh38eTAsfgDyDwQ6StMM+DOBrAL6ikiKiIQCVwNvn3LMmzh3H4hIPM4jrV1+jMmYavVJjObft46jXUQIs5/5ggff3URpRQtrneUPHXrClF/CDzbANa9C1zT47FF4dCi8eTtkbQ50hCaA/PoeiIhcADyKU7/xrKr+VkR+DaxW1bfFafbyR2A6UAn8VlVfqe2cVolu/Km4rJLfL9jMP1fsYUCnGP56zQj6JVkXIic5kg4rnnDqS8qLoO95cOZd0HOCvRnfjLWoVlj+YgnENIWPtmTys/nrKCip4OcXDOT6cT1adjNffyg6DKv+ASv/DkU5TtcqZ94JAy+2lxmbIUsgWAIxTSe7oJSfzv+aJVuzObt/An+YOYyEmLBAh9X8lBfD1y/D8sfg8E6I7eEkkuHXOZX3x1VWQEme8yZ98REnARUfcfr7qiiF9snQoYfz2Cwqwe5mfMwSCJZATNNSVf65Yg+/e38z0WHBPHLFUKYMSAp0WM1TVSVsfR8++4sz5klERychFB+BoiNQerT+5wqJhNjuTjLq0OPkeXC480JkRZk7L/3mJckK90XJilLnuO5joEOKJSMsgQCWQExgbMss4K6Xv2LLoQKuH9eDn18wsF6jKrZJqrD3c/jiKSjNd7peiejozCPd+Yltsc42Twgc3QdH9jgvNHrPj6RDWUHD42nXzamfSTkLep7lJKI2yBIIlkBM4JRWVPLIf7fyj093kxIfxXVjunPxsC7VjtFufEjVuYs5kg55e6Gqwn2TPuzk+anLpfmw5zPYvQzSP3XqacB5QfJ4Muk5oc0MLWwJBEsgJvCWbc/m4f9uYcP+fDwC4/vEc+nwrkxL7UR0mFUeN0uqTpPj9E8hfakzLz7i7OvQE0bPhTG3tequWyyBYAnENB87so7x1tr9vPHVfjKOFBMe4mHqoE7MGNGFs/omEBIU6I4eTI2qqiBrE6Qvgy3vOfPu4+CSx5038FshSyBYAjHNj6ry5d4jvPHVft5bd5AjReV0jArlf4Z05tIRXTmje6w1AW7OVJ3OJ9//mVMBP/UBGPUd8LSu/wBYAsESiGneyiqqWLotmzfX7ueDTZmUVlTRPiKEYcmxDO/WnmHJsQxLjiU+2poDNzv5B+Dtu2DHB079yCWPOY+3WglLIFgCMS1HQUk5H2zK5Ivdh1m7L49tmQVUuf/cunWIcJOKk1BSu7YjMtSpPykuqySroITsglKyCkrd+TfrOcdK6dw+grP6xjO+Tzy94qPsDsdXVJ1u8P97L2gVnPcbSLupVTQDtgSCJRDTchWVVbBhfz5f78tjrTsd7/03yCN0bh9OXlE5x0orvvXZII+QEB1GQkwYcdGh7Mg6RsYR57Nd2oczvk88E/rGc2bveHvZ0Rfy9sHbd8Kuj6HXZLj4sRbfWssSCJZATOuSXVDKuow8vt6Xx57DRXSMCiUhJozEmHB37iSNjpGheDzf/C9YVdl7uIhPd+Tw6fYclu/M5Wix09X6gE4xTHATyuiUjifubMxpUoU1z8HCX4J4YPrvYMTsFns3YgkESyDGVKeyStl44CjLtufw2Y4cVqcfoayyitAgD2N7x3HOgESmDEgkuWNk3SczJzuSDm/d4bTU6poGHVMgNApCo90pymvda7ljL2c8lmbCEgiWQIypj+KySlalH+aTbdl8vCWLXTmFAPRLimbKgCTOGZjIiORYgq2pcf1UVTkdR371IpQWQFmhM5UX1v659snQeRh0GQ6dRzjzqPgGxlDpdNkS0rAXVy2BYAnEmIbYlX2Mj7Zk8fHWLFbuOkxFldI+IoTJ/ROYMiCRyf0SaR8ZQkl5JXlF5RwtLievqIyjxeXfmgSY3D+RM/vEERbcel+8q5eqKqdL+7JCKDvmToVQkg85W+HAWji4Fg57DXN0alLpmOJ0LFmY7U5ZUJjzzfoxd16UCxN/4ozP0gCWQLAEYkxj5ZeU8+n2HD7c7CSUw4VleASCgzyUVVTV+DmPQLuIEMorqigsqyQmLJhzBiYyPbUzk/snWN9gtSnOg0PrvkkoB9Y6PRfXJKyd0yNxVIJzxxKd6Cz3nAApExsUgiUQLIEY40uVVcrXGXks2ZpNaUUl7SNCiI0IdeaRIbSPcKfIEKJDg/F4hNKKSpbvyGXBhoMs2pRJXlE5ESFBnD0ggempnZkyING6dKmPkqNwcB0czYDIuG8SRWR8gx9T1cYSCJZAjGlOKiqrWLn7MO+vP8jCjZnkHCslNNjDxL7xTE/tzKieHegaG2F1Lc2AJRAsgRjTXFVWKWv2HGHBhoP8d8MhDh4tASAkSEjuEEnP+Ch6xkWREv/NcpfYCII8LbNZbEtjCQRLIMa0BFVVyoYDR9lysIDduYWk5xSyO6eQPblFFJdXnjguNMhDcscIusRGoOokoUpVZ16lVKlSUenMj+/rnRDN5P4JTOqXQLcO1iy5vvyRQPz6oFJEpgN/AYKAf6jqQzUcdzkwHxilqpYdjGnhPB5haLdYhnaLPWm7qpJVUMruHDepuMklM7/Uqcj3ePB4ICTEQ5DHQ5A4b+EfnwC+3neUDzZlAtA7IYrJ/ROZ1C+B0Skd612Rr6ocLiwjPbeQvYeL6N4xihHJsSe9rGnq5rc7EBEJArYBU4EMYBVwjapuOuW4GOA9IBS4o64EYncgxrRtqsrO7EKWbM3ik23ZrNx9mLKKKsJDPIzrFcekfglM7p9Iz/go8orKnGSVW8junCLSTywXUlBycpcxCTFhnDswkfMGdWJc77hW16qsRT3CEpFxwP2qOs1dvxdAVX9/ynGPAh8APwV+YgnEGHM6issq+XxXLp9sy2bJ1izSc4sAiAoNorDsm8dlItA1NoIUt/6lZ7xTH5PcIZJNB/NZtCmTJVuyKCyrJCo0iEn9EzhvUCfO7u+8I9PStbRHWF2BfV7rGcAY7wNE5AwgWVXfE5Gf1nQiEZkLzAXo3r27H0I1xrRUEaFBnD0gkbMHJAKDSc8pZOn2bHZkHaNbhwh6xkXRKyGK5I6RNb742DcphkuGd6W0opIVO3NZtCmTDzZl8v76QwR7hDG9OjJ1YBLnDkqyehcv/rwDmQlMV9Vb3PXZwBhVvcNd9wAfAXNUNV1ElmB3IMaYZqLKfUdm0aZMFm08xM5sp9uSPonRTOybwMR+8YxJiSMitGU86mppdyD7Ae/+j7u5246LAVKBJe5YBp2At0XkYqtIN8YEmscjjOjegRHdO3D39AHszD7Gx1ucepd5K/fw7Ge7CQ32MCalo5tQEuiXFN2mxmbx5x1IME4l+jk4iWMVcK2qbqzh+CXYHYgxpgUoLqtk5e5clm7LOfG4DKBTu3DO6hvPxH4JpMRHERUWTFRoEFFhwUSGBgU0ubSoOxBVrRCRO4CFOM14n1XVjSLya2C1qr7tr7KNMcafIkKDmNw/kcn9EwE4kFfM0m3ZLN2ezcKNh3htTca3PiMCUaHBRIUFuYnFWY4JDyEuKpSO7hQXHUrHqDDiTiyHNttOK+1FQmOM8aGKyio2HMgnK7+EwrIKjpVWUlha4U7O8rEyZ72otJL8knJyC8s4UlhGRVX1f4+jw4LpGBXK9eN6cMtZvRoUV4u6AzHGmLYoOMjD8OTYug88haqSX1xBbmEphwvLyDlWxuHCMg4XlpJb6CzHRzev4YotgRhjTDMgIrSPdHo+7pUQ6Gjqx7rINMYY0yCWQIwxxjSIJRBjjDENYgnEGGNMg1gCMcYY0yCWQIwxxjSIJRBjjDENYgnEGGNMg7S4rkxEJBvY08CPxwM5PgynOWht19Targda3zW1tuuB1ndN1V1PD1X16SuKLS6BNIaIrPZ1XzCB1tquqbVdD7S+a2pt1wOt75qa6nrsEZYxxpgGsQRijDGmQdpaAnkq0AH4QWu7ptZ2PdD6rqm1XQ+0vmtqkutpU3UgxhhjfKet3YEYY4zxEUsgxhhjGqTNJBARmS4iW0Vkh4jcE+h4TiUi6SKyXkTWishqd1tHEflARLa78w7udhGRv7rXsk5EzvA6zw3u8dtF5Aav7SPd8+9wPyt+uIZnRSRLRDZ4bfP7NdRUhp+u534R2e9+T2tF5AKvffe6sW0VkWle26v93RORFBFZ6W5/VURC3e1h7voOd39PH11Psoh8LCKbRGSjiHzf3d6Sv6OarqlFfk8iEi4iX4jI1+71PNDQGHx1nbVS1VY/AUHATqAXEAp8DQwKdFynxJgOxJ+y7Q/APe7yPcDD7vIFwAJAgLHASnd7R2CXO+/gLndw933hHivuZ8/3wzVMBM4ANjTlNdRUhp+u537gJ9UcO8j9vQoDUtzft6DafveAfwNXu8t/A77rLt8O/M1dvhp41UfX0xk4w12OAba5cbfk76ima2qR35P7c4t2l0OAle7P87Ri8OV11hqvL77E5j4B44CFXuv3AvcGOq5TYkzn2wlkK9DZXe4MbHWX/w5cc+pxwDXA3722/93d1hnY4rX9pON8fB09OfkPrt+voaYy/HQ991P9H6aTfqeAhe7vXbW/e+4fihwg+NTf0eOfdZeD3ePED9/VW8DUlv4d1XBNLf57AiKBL4ExpxuDL6+ztqmtPMLqCuzzWs9wtzUnCiwSkTUiMtfdlqSqB93lQ0CSu1zT9dS2PaOa7U2hKa6hpjL85Q73kc6zXo9iTvd64oA8Va04ZftJ53L3H3WP9xn3UccInP/htorv6JRrghb6PYlIkIisBbKAD3DuGE43Bl9eZ43aSgJpCSao6hnA+cD3RGSi9051/lvQottcN8U1NEEZTwK9geHAQeCPfizLL0QkGngd+IGq5nvva6nfUTXX1GK/J1WtVNXhQDdgNDAgwCHVqK0kkP1Astd6N3dbs6Gq+915FvAGzi9Opoh0BnDnWe7hNV1Pbdu7VbO9KTTFNdRUhs+paqb7D7wKeBrne6KOuKvbngvEikjwKdtPOpe7v717fKOJSAjOH9qXVPU/7uYW/R1Vd00t/XtyryEP+BjncdLpxuDL66xRW0kgq4C+biuDUJzKprcDHNMJIhIlIjHHl4HzgA04MR5v4XIDzvNd3O3Xu61kxgJH3ccDC4HzRKSDe8t+Hs5zzINAvoiMdVvFXO91Ln9rimuoqQyfO/5H0DUD53s6HsPVbquYFKAvToVytb977v/CPwZmVhO39/XMBD5yj29s7AI8A2xW1T957Wqx31FN19RSvycRSRCRWHc5Aqc+Z3MDYvDlddbMHxVZzXHCaVGyDed54i8CHc8psfXCaQ3xNbDxeHw4zyU/BLYDi4GO7nYBHnevZT2Q5nWum4Ad7nSj1/Y0nH9EO4HH8E+l7Ms4jwvKcZ6h3twU11BTGX66nhfdeNe5/0g7ex3/Cze2rXi1cqvpd8/93r9wr/M1IMzdHu6u73D39/LR9UzAeXS0DljrThe08O+opmtqkd8TMBT4yo17A/D/GhqDr66ztsm6MjHGGNMgbeURljHGGB+zBGKMMaZBLIEYY4xpEEsgxhhjGsQSiDHGmAaxBGLaJBGplG96al0rdfTQLCK3icj1Pig3XUTiG3seY5oDa8Zr2iQROaaq0QEoNx3nfYqcpi7bGF+zOxBjvLh3CH8QZ0yLL0Skj7v9fhH5ibt8lzjjT6wTkVfcbR1F5E132+ciMtTdHicii8QZ2+EfOC/nHS9rllvGWhH5u9uJXpCIPC8iG9wYfhiAH4Mx9WIJxLRVEac8wrrKa99RVR2C8yb1o9V89h5ghKoOBW5ztz0AfOVu+znwT3f7fcCnqjoYp4+z7gAiMhC4ChivTsd5lcB1OJ3/dVXVVDeG53x4zcb4VHDdhxjTKhW7f7ir87LX/M/V7F8HvCQibwJvutsmAJcDqOpH7p1HO5xBqS5zt78nIkfc488BRgKrnO6ciMDpYPAdoJeI/B/wHrCo4ZdojH/ZHYgx36Y1LB/3Pzh9RJ2BkwAa8h8xAV5Q1eHu1F9V71fVI8AwYAnO3c0/GnBuY5qEJRBjvu0qr/kK7x0i4gGSVfVj4G6c7rOjgWU4j6AQkclAjjrjUiwFrnW3n48zBCw4HQvOFJFEd19HEenhttDyqOrrwC9xkpQxzZI9wjJtVYQ4o74d919VPd6Ut4OIrANKcYZl9RYEzJP/394d2iAYxGAA/coCDMoKCFiBAUCyCRvAAEyAAnmI+xOQpArxnmxy+ksvTVu1zuwiDmOMR1XtkhyXd898Vmzvk5yr6prkkuSeJGOMW1VtM69QrjI3/m6SvJKclloyz43CXzLGC1+M2cLvfGEB0KIDAaBFBwJAiwABoEWAANAiQABoESAAtLwBD4zlyV0vQ9gAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = [i * 10000 for i in range(len(tf_leduc_result))]\n", + "\n", + "plt.plot(x, tf_leduc_result, label='tensorflow')\n", + "plt.plot(x, pt_leduc_result, label='pytorch')\n", + "plt.title('Leduc Poker')\n", + "plt.xlabel('Episodes')\n", + "plt.ylabel('Exploitability')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/open_spiel/python/CMakeLists.txt b/open_spiel/python/CMakeLists.txt index 6bbf03600e..cdd8989204 100644 --- a/open_spiel/python/CMakeLists.txt +++ b/open_spiel/python/CMakeLists.txt @@ -155,6 +155,7 @@ set(PYTHON_TESTS ${PYTHON_TESTS} utils/spawn_test.py pytorch/rcfr_pytorch_test.py pytorch/dqn_pytorch_test.py + pytorch/nfsp_pytorch_test.py ) diff --git a/open_spiel/python/pytorch/nfsp.py b/open_spiel/python/pytorch/nfsp.py new file mode 100644 index 0000000000..9d52730022 --- /dev/null +++ b/open_spiel/python/pytorch/nfsp.py @@ -0,0 +1,343 @@ +# Copyright 2019 DeepMind Technologies Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Neural Fictitious Self-Play (NFSP) agent implemented in PyTorch. + +See the paper https://arxiv.org/abs/1603.01121 for more details. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import contextlib +import enum +import os +import random +from absl import logging +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from open_spiel.python import rl_agent +from open_spiel.python.pytorch import dqn + + +Transition = collections.namedtuple( + "Transition", "info_state action_probs legal_actions_mask") + +ILLEGAL_ACTION_LOGITS_PENALTY = -1e9 + +MODE = enum.Enum("mode", "best_response average_policy") + + +class NFSP(rl_agent.AbstractAgent): + """NFSP Agent implementation in PyTorch. + + See open_spiel/python/examples/kuhn_nfsp.py for an usage example. + """ + + def __init__(self, + player_id, + state_representation_size, + num_actions, + hidden_layers_sizes, + reservoir_buffer_capacity, + anticipatory_param, + batch_size=128, + rl_learning_rate=0.01, + sl_learning_rate=0.01, + min_buffer_size_to_learn=1000, + learn_every=64, + optimizer_str="sgd", + **kwargs): + """Initialize the `NFSP` agent.""" + self.player_id = player_id + self._num_actions = num_actions + self._layer_sizes = hidden_layers_sizes + self._batch_size = batch_size + self._learn_every = learn_every + self._anticipatory_param = anticipatory_param + self._min_buffer_size_to_learn = min_buffer_size_to_learn + + self._reservoir_buffer = ReservoirBuffer(reservoir_buffer_capacity) + self._prev_timestep = None + self._prev_action = None + + # Step counter to keep track of learning. + self._step_counter = 0 + + # Inner RL agent + kwargs.update({ + "batch_size": batch_size, + "learning_rate": rl_learning_rate, + "learn_every": learn_every, + "min_buffer_size_to_learn": min_buffer_size_to_learn, + "optimizer_str": optimizer_str, + }) + self._rl_agent = dqn.DQN(player_id, state_representation_size, + num_actions, hidden_layers_sizes, **kwargs) + + # Keep track of the last training loss achieved in an update step. + self._last_rl_loss_value = lambda: self._rl_agent.loss + self._last_sl_loss_value = None + + # Average policy network. + self._avg_network = dqn.MLP(state_representation_size, + self._layer_sizes, num_actions) + + self._savers = [ + ("q_network", self._rl_agent._q_network), + ("avg_network", self._avg_network) + ] + + if optimizer_str == "adam": + self.optimizer = torch.optim.Adam( + self._avg_network.parameters(), lr=sl_learning_rate) + elif optimizer_str == "sgd": + self.optimizer = torch.optim.SGD( + self._avg_network.parameters(), lr=sl_learning_rate) + else: + raise ValueError("Not implemented. Choose from ['adam', 'sgd'].") + + self._sample_episode_policy() + + @contextlib.contextmanager + def temp_mode_as(self, mode): + """Context manager to temporarily overwrite the mode.""" + previous_mode = self._mode + self._mode = mode + yield + self._mode = previous_mode + + def _sample_episode_policy(self): + if np.random.rand() < self._anticipatory_param: + self._mode = MODE.best_response + else: + self._mode = MODE.average_policy + + def _act(self, info_state, legal_actions): + info_state = np.reshape(info_state, [1, -1]) + action_values = self._avg_network(torch.Tensor(info_state)) + action_probs = F.softmax(action_values, dim=1).detach() + + self._last_action_values = action_values[0] + # Remove illegal actions, normalize probs + probs = np.zeros(self._num_actions) + probs[legal_actions] = action_probs[0][legal_actions] + probs /= sum(probs) + action = np.random.choice(len(probs), p=probs) + return action, probs + + @property + def mode(self): + return self._mode + + @property + def loss(self): + return (self._last_sl_loss_value, self._last_rl_loss_value().detach()) + + def step(self, time_step, is_evaluation=False): + """Returns the action to be taken and updates the Q-networks if needed. + + Args: + time_step: an instance of rl_environment.TimeStep. + is_evaluation: bool, whether this is a training or evaluation call. + + Returns: + A `rl_agent.StepOutput` containing the action probs and chosen action. + """ + if self._mode == MODE.best_response: + agent_output = self._rl_agent.step(time_step, is_evaluation) + if not is_evaluation and not time_step.last(): + self._add_transition(time_step, agent_output) + + elif self._mode == MODE.average_policy: + # Act step: don't act at terminal info states. + if not time_step.last(): + info_state = time_step.observations["info_state"][self.player_id] + legal_actions = time_step.observations["legal_actions"][self.player_id] + action, probs = self._act(info_state, legal_actions) + agent_output = rl_agent.StepOutput(action=action, probs=probs) + + if self._prev_timestep and not is_evaluation: + self._rl_agent.add_transition(self._prev_timestep, self._prev_action, + time_step) + else: + raise ValueError("Invalid mode ({})".format(self._mode)) + + if not is_evaluation: + self._step_counter += 1 + + if self._step_counter % self._learn_every == 0: + self._last_sl_loss_value = self._learn() + # If learn step not triggered by rl policy, learn. + if self._mode == MODE.average_policy: + self._rl_agent.learn() + + # Prepare for the next episode. + if time_step.last(): + self._sample_episode_policy() + self._prev_timestep = None + self._prev_action = None + return + else: + self._prev_timestep = time_step + self._prev_action = agent_output.action + + return agent_output + + def _add_transition(self, time_step, agent_output): + """Adds the new transition using `time_step` to the reservoir buffer. + + Transitions are in the form (time_step, agent_output.probs, legal_mask). + + Args: + time_step: an instance of rl_environment.TimeStep. + agent_output: an instance of rl_agent.StepOutput. + """ + legal_actions = time_step.observations["legal_actions"][self.player_id] + legal_actions_mask = np.zeros(self._num_actions) + legal_actions_mask[legal_actions] = 1.0 + transition = Transition( + info_state=(time_step.observations["info_state"][self.player_id][:]), + action_probs=agent_output.probs, + legal_actions_mask=legal_actions_mask) + self._reservoir_buffer.add(transition) + + def _learn(self): + """Compute the loss on sampled transitions and perform a avg-network update. + + If there are not enough elements in the buffer, no loss is computed and + `None` is returned instead. + + Returns: + The average loss obtained on this batch of transitions or `None`. + """ + if (len(self._reservoir_buffer) < self._batch_size or + len(self._reservoir_buffer) < self._min_buffer_size_to_learn): + return None + + transitions = self._reservoir_buffer.sample(self._batch_size) + info_states = torch.Tensor([t.info_state for t in transitions]) + action_probs = torch.LongTensor([t.action_probs for t in transitions]) + legal_actions_mask = torch.Tensor([t.legal_actions_mask for t in transitions]) + + self.optimizer.zero_grad() + loss = F.cross_entropy(self._avg_network(info_states), torch.max(action_probs, dim=1)[1]) + loss.backward() + self.optimizer.step() + return loss.detach() + + def _full_checkpoint_name(self, checkpoint_dir, name): + checkpoint_filename = "_".join([name, "pid" + str(self.player_id)]) + return os.path.join(checkpoint_dir, checkpoint_filename) + + def _latest_checkpoint_filename(self, name): + checkpoint_filename = "_".join([name, "pid" + str(self.player_id)]) + return checkpoint_filename + "_latest" + + def save(self, checkpoint_dir): + """Saves the average policy network and the inner RL agent's q-network. + + Note that this does not save the experience replay buffers and should + only be used to restore the agent's policy, not resume training. + + Args: + checkpoint_dir: directory where checkpoints will be saved. + """ + for name, model in self._savers: + path = self._full_checkpoint_name(checkpoint_dir, name) + torch.save(model.state_dict(), path) + logging.info("Saved to path: %s", path) + + def has_checkpoint(self, checkpoint_dir): + for name, _ in self._savers: + path = self._full_checkpoint_name(checkpoint_dir, name) + if os.path.exists(path): + return True + return False + + def restore(self, checkpoint_dir): + """Restores the average policy network and the inner RL agent's q-network. + + Note that this does not restore the experience replay buffers and should + only be used to restore the agent's policy, not resume training. + + Args: + checkpoint_dir: directory from which checkpoints will be restored. + """ + for name, model in self._savers: + full_checkpoint_dir = self._full_checkpoint_name(checkpoint_dir, name) + logging.info("Restoring checkpoint: %s", full_checkpoint_dir) + mdoel.load_state_dict(torch.load(full_checkpoint_dir)) + + +class ReservoirBuffer(object): + """Allows uniform sampling over a stream of data. + + This class supports the storage of arbitrary elements, such as observation + tensors, integer actions, etc. + + See https://en.wikipedia.org/wiki/Reservoir_sampling for more details. + """ + + def __init__(self, reservoir_buffer_capacity): + self._reservoir_buffer_capacity = reservoir_buffer_capacity + self._data = [] + self._add_calls = 0 + + def add(self, element): + """Potentially adds `element` to the reservoir buffer. + + Args: + element: data to be added to the reservoir buffer. + """ + if len(self._data) < self._reservoir_buffer_capacity: + self._data.append(element) + else: + idx = np.random.randint(0, self._add_calls + 1) + if idx < self._reservoir_buffer_capacity: + self._data[idx] = element + self._add_calls += 1 + + def sample(self, num_samples): + """Returns `num_samples` uniformly sampled from the buffer. + + Args: + num_samples: `int`, number of samples to draw. + + Returns: + An iterable over `num_samples` random elements of the buffer. + + Raises: + ValueError: If there are less than `num_samples` elements in the buffer + """ + if len(self._data) < num_samples: + raise ValueError("{} elements could not be sampled from size {}".format( + num_samples, len(self._data))) + return random.sample(self._data, num_samples) + + def clear(self): + self._data = [] + self._add_calls = 0 + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) diff --git a/open_spiel/python/pytorch/nfsp_pytorch_test.py b/open_spiel/python/pytorch/nfsp_pytorch_test.py new file mode 100644 index 0000000000..e80916c0c2 --- /dev/null +++ b/open_spiel/python/pytorch/nfsp_pytorch_test.py @@ -0,0 +1,111 @@ +# Copyright 2019 DeepMind Technologies Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for open_spiel.python.algorithms.nfsp.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +from scipy import stats + +from torch.testing._internal.common_utils import run_tests +from torch.testing._internal.common_utils import TestCase + +from open_spiel.python import rl_environment +from open_spiel.python.pytorch import nfsp + + +class NFSPTest(TestCase): + + def test_run_kuhn(self): + env = rl_environment.Environment("kuhn_poker") + state_size = env.observation_spec()["info_state"][0] + num_actions = env.action_spec()["num_actions"] + + agents = [ + nfsp.NFSP( # pylint: disable=g-complex-comprehension + player_id, + state_representation_size=state_size, + num_actions=num_actions, + hidden_layers_sizes=[16], + reservoir_buffer_capacity=10, + anticipatory_param=0.1) for player_id in [0, 1] + ] + for unused_ep in range(10): + time_step = env.reset() + while not time_step.last(): + current_player = time_step.observations["current_player"] + current_agent = agents[current_player] + agent_output = current_agent.step(time_step) + time_step = env.step([agent_output.action]) + for agent in agents: + agent.step(time_step) + +class ReservoirBufferTest(TestCase): + + def test_reservoir_buffer_add(self): + reservoir_buffer = nfsp.ReservoirBuffer(reservoir_buffer_capacity=10) + self.assertEqual(len(reservoir_buffer), 0) + reservoir_buffer.add("entry1") + self.assertEqual(len(reservoir_buffer), 1) + reservoir_buffer.add("entry2") + self.assertEqual(len(reservoir_buffer), 2) + + self.assertIn("entry1", reservoir_buffer) + self.assertIn("entry2", reservoir_buffer) + + def test_reservoir_buffer_max_capacity(self): + reservoir_buffer = nfsp.ReservoirBuffer(reservoir_buffer_capacity=2) + reservoir_buffer.add("entry1") + reservoir_buffer.add("entry2") + reservoir_buffer.add("entry3") + + self.assertEqual(len(reservoir_buffer), 2) + + def test_reservoir_uniform(self): + size = 10 + max_value = 100 + num_trials = 1000 + expected_count = 1. / max_value * size * num_trials + + reservoir_buffer = nfsp.ReservoirBuffer(reservoir_buffer_capacity=size) + counter = collections.Counter() + for _ in range(num_trials): + reservoir_buffer.clear() + for idx in range(max_value): + reservoir_buffer.add(idx) + data = reservoir_buffer.sample(size) + counter.update(data) + # Tests the null hypothesis (H0) that data has the given frequencies. + # We reject the null hypothesis if we get a p-value below our threshold. + pvalue = stats.chisquare(list(counter.values()), expected_count).pvalue + self.assertGreater(pvalue, 0.05) # We cannot reject H0. + + def test_reservoir_buffer_sample(self): + replay_buffer = nfsp.ReservoirBuffer(reservoir_buffer_capacity=3) + replay_buffer.add("entry1") + replay_buffer.add("entry2") + replay_buffer.add("entry3") + + samples = replay_buffer.sample(3) + + self.assertIn("entry1", samples) + self.assertIn("entry2", samples) + self.assertIn("entry3", samples) + + +if __name__ == "__main__": + run_tests()