iql cql dreamer impala benchmarks

pytorch · vmoens · Jan 31, 2024 · Jan 9, 2024 · Jan 18, 2024 · Jan 18, 2024
commit e0681ce67d532da40458cecb9056e052dfd2cab9
diff --git a/benchmarks/sota-check/run_cql_offline.sh b/benchmarks/sota-check/run_cql_offline.sh
@@ -1,10 +1,15 @@
 #!/bin/bash
 
 #SBATCH --job-name=cql_offline
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=cql_offline_output_%j.txt
 #SBATCH --error=cql_offline_error_%j.txt
 
-python ../../examples/cql/cql_offline.py
+python ../../examples/cql/cql_offline.py \
+  optim.gradient_steps=55 \
+  optim.device=cuda:0 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_cql_online.sh b/benchmarks/sota-check/run_cql_online.sh
@@ -1,10 +1,19 @@
 #!/bin/bash
 
 #SBATCH --job-name=cql_online
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=cql_online_output_%j.txt
 #SBATCH --error=cql_online_error_%j.txt
 
-python ../../examples/cql/cql_online.py
+python ../../examples/cql/cql_online.py \
+  collector.total_frames=48 \
+  optim.batch_size=10 \
+  collector.frames_per_batch=16 \
+  collector.env_per_collector=1 \
+  optim.device=cuda:0 \
+  collector.device=cuda:0 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_ddpg.sh b/benchmarks/sota-check/run_ddpg.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 #SBATCH --job-name=ddpg
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1

diff --git a/benchmarks/sota-check/run_dreamer.sh b/benchmarks/sota-check/run_dreamer.sh
@@ -1,10 +1,27 @@
 #!/bin/bash
 
 #SBATCH --job-name=dreamer
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=dreamer_output_%j.txt
 #SBATCH --error=dreamer_error_%j.txt
 
-python ../../examples/dreamer/dreamer.py
+python ../../examples/dreamer/dreamer.py \
+  total_frames=200 \
+  init_random_frames=10 \
+  batch_size=10 \
+  frames_per_batch=200 \
+  num_workers=4 \
+  env_per_collector=2 \
+  collector_device=cuda:0 \
+  model_device=cuda:0 \
+  optim_steps_per_batch=1 \
+  record_video=True \
+  record_frames=4 \
+  buffer_size=120 \
+  rssm_hidden_dim=17 \
+  logger=wandb \
+  project_name="sota-check"
+
diff --git a/benchmarks/sota-check/run_dt.sh b/benchmarks/sota-check/run_dt.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#SBATCH --job-name=dt
+#SBATCH --partition=test
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=dt_offline_output_%j.txt
+#SBATCH --error=dt_offline_error_%j.txt
+
+python ../../examples/dt/dt.py \
+  optim.pretrain_gradient_steps=55 \
+  optim.updates_per_episode=3 \
+  optim.warmup_steps=10 \
+  optim.device=cuda:0 \
+  env.backend=gymnasium \
+  env.name=HalfCheetah-v4 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_dt_offline.sh b/benchmarks/sota-check/run_dt_offline.sh
diff --git a/benchmarks/sota-check/run_dt_online.sh b/benchmarks/sota-check/run_dt_online.sh
@@ -1,10 +1,17 @@
 #!/bin/bash
 
 #SBATCH --job-name=dt_online
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=dt_online_output_%j.txt
 #SBATCH --error=dt_online_error_%j.txt
 
-python ../../examples/dt/dt_online.py
+python ../../examples/dt/dt_online.py \
+  optim.pretrain_gradient_steps=55 \
+  optim.updates_per_episode=3 \
+  optim.warmup_steps=10 \
+  optim.device=cuda:0 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_impala.sh b/benchmarks/sota-check/run_impala.sh
diff --git a/benchmarks/sota-check/run_impala_single_node.sh b/benchmarks/sota-check/run_impala_single_node.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH --job-name=impala_1node
+#SBATCH --partition=test
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=impala_1node_output_%j.txt
+#SBATCH --error=impala_1node_error_%j.txt
+
+python ../../examples/impala/impala_single_node.py \
+  collector.total_frames=80 \
+  collector.frames_per_batch=20 \
+  collector.num_workers=1 \
+  logger.test_interval=10 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_iql_offline.sh b/benchmarks/sota-check/run_iql_offline.sh
@@ -1,10 +1,15 @@
 #!/bin/bash
 
 #SBATCH --job-name=iql_offline
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=iql_offline_output_%j.txt
 #SBATCH --error=iql_offline_error_%j.txt
 
-python ../../examples/iql/iql_offline.py
+python ../../examples/iql/iql_offline.py \
+  optim.gradient_steps=55 \
+  optim.device=cuda:0 \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_iql_online.sh b/benchmarks/sota-check/run_iql_online.sh
@@ -1,10 +1,20 @@
 #!/bin/bash
 
 #SBATCH --job-name=iql_online
+#SBATCH --partition=test
 #SBATCH --ntasks=32
 #SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=iql_online_output_%j.txt
 #SBATCH --error=iql_online_error_%j.txt
 
-python ../../examples/iql/iql_online.py
+python ../../examples/iql/iql_online.py \
+  collector.total_frames=48 \
+  optim.batch_size=10 \
+  collector.frames_per_batch=16 \
+  env.train_num_envs=2 \
+  optim.device=cuda:0 \
+  collector.device=cuda:0 \
+  logger.mode=offline \
+  logger.backend=wandb \
+  logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/run_redq.sh b/benchmarks/sota-check/run_redq.sh
@@ -8,7 +8,7 @@
 #SBATCH --output=redq_output_%j.txt
 #SBATCH --error=redq_error_%j.txt
 
-CUDA_VISIBLE_DEVICES=1 python ../../examples/redq/redq.py \
+python ../../examples/redq/redq.py \
   num_workers=2 \
   collector.total_frames=48 \
   collector.init_random_frames=10 \

diff --git a/benchmarks/sota-check/run_td3.sh b/benchmarks/sota-check/run_td3.sh
@@ -18,7 +18,6 @@ python ../../examples/td3/td3.py \
   collector.device=cuda:0 \
   collector.device=cuda:0 \
   network.device=cuda:0 \
-  logger.mode=offline \
   env.name=Pendulum-v1 \
   logger.backend=wandb \
   logger.project_name="sota-check"
diff --git a/benchmarks/sota-check/submitit-release-check.sh b/benchmarks/sota-check/submitit-release-check.sh
@@ -3,18 +3,18 @@
 sbatch run_a2c_atari.sh
 sbatch run_a2c_mujoco.sh
 #sbatch run_bandits.sh
-#sbatch run_cql_offline.sh
-#sbatch run_cql_online.sh
+sbatch run_cql_offline.sh
+sbatch run_cql_online.sh
 sbatch run_ddpg.sh
 sbatch run_discrete_sac.sh
 sbatch run_dqn_atari.sh
 sbatch run_dqn_cartpole.sh
-#sbatch run_dreamer.sh
-#sbatch run_dt_offline.sh
-#sbatch run_dt_online.sh
-#sbatch run_impala.sh
-#sbatch run_iql_offline.sh
-#sbatch run_iql_online.sh
+sbatch run_dreamer.sh
+sbatch run_dt.sh
+sbatch run_dt_online.sh
+sbatch run_impala_single_node.sh
+sbatch run_iql_offline.sh
+sbatch run_iql_online.sh
 #sbatch run_multiagent_iddpg.sh
 #sbatch run_multiagent_ippo.sh
 #sbatch run_multiagent_iql.sh

diff --git a/examples/cql/cql_offline.py b/examples/cql/cql_offline.py
@@ -39,7 +39,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
             logger_type=cfg.logger.backend,
             logger_name="cql_logging",
             experiment_name=exp_name,
-            wandb_kwargs={"mode": cfg.logger.mode, "config": cfg},
+            wandb_kwargs={"mode": cfg.logger.mode, "config": dict(cfg), "project": cfg.logger.project_name},
         )
     # Set seeds
     torch.manual_seed(cfg.env.seed)

diff --git a/examples/cql/cql_online.py b/examples/cql/cql_online.py
@@ -43,7 +43,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
             logger_type=cfg.logger.backend,
             logger_name="cql_logging",
             experiment_name=exp_name,
-            wandb_kwargs={"mode": cfg.logger.mode, "config": cfg},
+            wandb_kwargs={"mode": cfg.logger.mode, "config": dict(cfg), "project": cfg.logger.project_name},
         )
 
     # Set seeds

diff --git a/examples/cql/discrete_cql_config.yaml b/examples/cql/discrete_cql_config.yaml
@@ -24,6 +24,7 @@ collector:
 # Logger
 logger:
   backend: wandb
+  project_name: rl_examples_cql
   log_interval: 5000 # record interval in frames
   eval_steps: 200
   mode: online

diff --git a/examples/cql/discrete_cql_online.py b/examples/cql/discrete_cql_online.py
@@ -45,7 +45,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
             logger_type=cfg.logger.backend,
             logger_name="discretecql_logging",
             experiment_name=exp_name,
-            wandb_kwargs={"mode": cfg.logger.mode, "config": cfg},
+            wandb_kwargs={"mode": cfg.logger.mode, "config": dict(cfg), "project": cfg.logger.project_name},
         )
 
     # Set seeds

diff --git a/examples/cql/offline_config.yaml b/examples/cql/offline_config.yaml
@@ -11,6 +11,7 @@ env:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_cql
   eval_iter: 5000
   eval_steps: 1000
   mode: online

diff --git a/examples/cql/online_config.yaml b/examples/cql/online_config.yaml
@@ -23,6 +23,7 @@ collector:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_cql
   log_interval: 5000 # record interval in frames
   eval_steps: 1000
   mode: online

diff --git a/examples/decision_transformer/dt_config.yaml b/examples/decision_transformer/dt_config.yaml
@@ -20,6 +20,7 @@ env:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_dt
   model_name: DT
   exp_name: DT-HalfCheetah-medium-v2
   pretrain_log_interval: 500 # record interval in frames

diff --git a/examples/decision_transformer/odt_config.yaml b/examples/decision_transformer/odt_config.yaml
@@ -20,6 +20,7 @@ env:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_odt
   exp_name: oDT-HalfCheetah-medium-v2
   model_name: oDT
   pretrain_log_interval: 500 # record interval in frames

diff --git a/examples/decision_transformer/utils.py b/examples/decision_transformer/utils.py
@@ -496,7 +496,7 @@ def make_logger(cfg):
         cfg.logger.backend,
         logger_name=cfg.logger.model_name,
         experiment_name=exp_name,
-        wandb_kwargs={"config": OmegaConf.to_container(cfg)},
+        wandb_kwargs={"config": dict(cfg), "project": cfg.logger.project_name},
     )
     return logger
 

diff --git a/examples/dreamer/config.yaml b/examples/dreamer/config.yaml
@@ -32,6 +32,7 @@ init_env_steps: 1000
 init_random_frames: 5000
 logger: csv
 offline_logging: False
+project_name: rl_examples_dreamer
 normalize_rewards_online: True
 normalize_rewards_online_scale: 5.0
 normalize_rewards_online_decay: 0.99999

diff --git a/examples/dreamer/dreamer.py b/examples/dreamer/dreamer.py
@@ -91,7 +91,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
         logger_name="dreamer",
         experiment_name=exp_name,
         wandb_kwargs={
-            "project": "torchrl",
+            "project": cfg.project_name,
             "group": f"Dreamer_{cfg.env_name}",
             "offline": cfg.offline_logging,
         },

diff --git a/examples/iql/iql_offline.py b/examples/iql/iql_offline.py
@@ -42,7 +42,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
             logger_type=cfg.logger.backend,
             logger_name="iql_logging",
             experiment_name=exp_name,
-            wandb_kwargs={"mode": cfg.logger.mode, "config": cfg},
+            wandb_kwargs={"mode": cfg.logger.mode, "config": dict(cfg), "project": cfg.logger.project_name},
         )
 
     # Set seeds

diff --git a/examples/iql/iql_online.py b/examples/iql/iql_online.py
@@ -43,7 +43,7 @@ def main(cfg: "DictConfig"):  # noqa: F821
             logger_type=cfg.logger.backend,
             logger_name="iql_logging",
             experiment_name=exp_name,
-            wandb_kwargs={"mode": cfg.logger.mode, "config": cfg},
+            wandb_kwargs={"mode": cfg.logger.mode, "config": dict(cfg), "project": cfg.logger.project_name},
         )
 
     # Set seeds

diff --git a/examples/iql/offline_config.yaml b/examples/iql/offline_config.yaml
@@ -10,6 +10,7 @@ env:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_iql
   eval_iter: 500
   eval_steps: 1000
   mode: online

diff --git a/examples/iql/online_config.yaml b/examples/iql/online_config.yaml
@@ -23,6 +23,7 @@ collector:
 # logger
 logger:
   backend: wandb
+  project_name: rl_examples_iql
   log_interval: 5000 # record interval in frames
   eval_steps: 200
   mode: online