Sets up whole-board runs

andyljones · Apr 15, 2021 · fda447a · fda447a
1 parent 85ea584
commit fda447a
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 9 deletions.
diff --git a/analysis/data.py b/analysis/data.py
@@ -35,7 +35,7 @@ def trial_elos(boardsize):
     return _trial_elos(boardsize, counter)
 
 def load():
-    ags = sql.agent_query().query('c == 1/16')
+    ags = sql.agent_query().query('test_c == 1/16')
 
     es = []
     for b in tqdm(ags.boardsize.unique()):

diff --git a/boardlaw/noisescales.py b/boardlaw/noisescales.py
@@ -1,3 +1,4 @@
+from rebar import parallel
 import pandas as pd
 import torch
 from .arena import common
@@ -6,9 +7,20 @@
 import numpy as np
 from tqdm.auto import tqdm
 from logging import getLogger
+from multiprocessing import set_start_method
 
 log = getLogger(__name__)
 
+# Smallest networks to make it to within -.5 of perfect play.
+RUNS = {
+    3: '2021-02-17 19-34-03 valid-ships',
+    4: '2021-02-21 05-22-28 watery-drunks',
+    5: '2021-02-19 11-15-16 skinny-tactic',
+    6: '2021-02-21 08-52-28 these-plow',
+    7: '2021-02-19 19-50-36 tan-buffer',
+    8: '2021-02-21 22-55-52 double-spoon',
+    9: '2021-02-20 21-11-32 intent-nets'}
+
 def stored_agent(agent_id):
     info = sql.query('select * from agents_details where id == ?', int(agent_id)).iloc[0]
     agent = common.agent(info.run, info.idx, 'cuda')
@@ -95,8 +107,8 @@ def noise_scale(result):
     return result.batch_size*result.variance/result.mean_sq
 
 def evaluate_noise_scale(agent_id):
-    extant = sql.query('select * from noise_scales where id == ?', agent_id)
-    if agent_id not in extant.index:
+    extant = sql.query('select * from noise_scales where id == ?', int(agent_id))
+    if agent_id not in extant.id.values:
         result = noise_scale_components(agent_id)
         log.info(f'{agent_id}: {noise_scale(result):.0f}')
         sql.save_noise_scale(result)
@@ -114,19 +126,20 @@ def evaluate_perf(agent_id, n_envs=1024):
     extant = sql.query('''
         select * from trials 
         where ((black_agent == ?) and (white_agent == ?)) 
-        or ((white_agent == ?) and (black_agent == ?))''', agent_id, opponent_id, agent_id, opponent_id)
+        or ((white_agent == ?) and (black_agent == ?))''', 
+        int(agent_id), int(opponent_id), int(agent_id), int(opponent_id))
     games = (extant.black_wins + extant.white_wins).sum()
     if games < n_envs:
         a = stored_agent(agent_id)
         o = stored_agent(opponent_id)
         w = stored_worlds(agent_id, n_envs)
 
-        results = common.evaluate(w, {agent_id: a, opponent_id: o})
+        results = common.evaluate(w, [(agent_id, a), (opponent_id, o)])
 
         sql.save_trials(results)
 
 def evaluate(run, idx, nodes, c_puct):
-    snap_id = sql.query_one('select id from snaps where run == ? and idx == ?', run, idx).id
+    snap_id = sql.query_one('select id from snaps where run == ? and idx == ?', run, int(idx)).id
     extant = sql.query('select * from agents where snap == ? and nodes == ? and c == ?', int(snap_id), int(nodes), float(c_puct))
     if len(extant) == 0:
         log.info(f'Creating agent run="{run}", idx={idx}, nodes={nodes}, c_puct={c_puct:.3f}')
@@ -137,6 +150,23 @@ def evaluate(run, idx, nodes, c_puct):
     evaluate_noise_scale(agent_id)
     evaluate_perf(agent_id)
 
+def evaluate_board(boardsize=None):
+    if boardsize is None:
+        for b in RUNS:
+            evaluate_board(b)
+        return 
+    run = RUNS[boardsize]
+    snaps = sql.query('select * from snaps where run == ?', run)
+
+    set_start_method('spawn', True)
+    with parallel.parallel(evaluate, N=2, executor='cuda', desc=str(boardsize)) as pool:
+        futures = {}
+        for idx in snaps.idx.unique():
+            for nodes in [1, 2, 4, 8, 16, 32, 64, 128, 256]:
+                for c in [1/64, 1/32, 1/16, 1/8, 1/4, 1/2, 1.]:
+                    futures[idx, nodes, c] = pool(run, idx, nodes, c)
+        pool.wait(futures)
+
 def load():
     from analysis import data
 

diff --git a/boardlaw/sql.py b/boardlaw/sql.py
@@ -150,7 +150,7 @@ class MohexTrial(Base):
 class NoiseScale(Base):
     __tablename__ = 'noise_scales'
 
-    id = Column(Integer, ForeignKey('snaps.id'), primary_key=True)
+    id = Column(Integer, ForeignKey('agents.id'), primary_key=True)
     mean_sq = Column(Float)
     sq_mean = Column(Float)
     variance = Column(Float)

diff --git a/rebar/parallel.py b/rebar/parallel.py
@@ -83,7 +83,7 @@ def VariableExecutor(N=None, executor='process', **kwargs):
 
 
 @contextmanager
-def parallel(f, progress=True, **kwargs):
+def parallel(f, progress=True, desc=None, **kwargs):
     """Sugar for using the VariableExecutor. Call as
     
     with parallel(f) as g:
@@ -121,7 +121,7 @@ def wait(c):
             futures = {fut: k for k, fut in c.items()}
 
             results = {}
-            for fut in tqdm(as_completed(futures), total=len(c), disable=not progress):
+            for fut in tqdm(as_completed(futures), total=len(c), disable=not progress, desc=desc):
                 results[futures[fut]] = reraise(fut, futures)
 
             return results