pytorch · vmoens · Jan 31, 2024 · Jan 9, 2024 · Jan 18, 2024 · Jan 18, 2024
diff --git a/benchmarks/sota-check/README.md b/benchmarks/sota-check/README.md
@@ -0,0 +1,35 @@
+# SOTA Performance checks
+
+This folder contains a `submitit-release-check.sh` file that executes all
+the training scripts using `sbatch` with the default configuration and long them
+into a common WandB project.
+
+This script is to be executed before every release to assess the performance of
+the various algorithms available in torchrl. The name of the project will include
+the specific commit of torchrl used to run the scripts (e.g. `torchrl-examples-check-<commit>`).
+
+## Usage
+
+To display the script usage, you can use the `--help` option:
+
+```bash
+./submitit-release-check.sh --help
+```
+
+## Setup
+
+The following setup should allow you to run the scripts:
+
+```bash
+export MUJOCO_GL=egl
+
+conda create -n rl-sota-bench python=3.10 -y 
+conda install anaconda::libglu -y
+pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121
+pip3 install "gymnasium[accept-rom-license,atari,mujoco]" vmas tqdm wandb pygame moviepy imageio submitit hydra-core transformers
+
+cd /path/to/tensordict
+python setup.py develop
+cd /path/to/torchrl
+python setup.py develop
+```
diff --git a/benchmarks/sota-check/run_a2c_atari.sh b/benchmarks/sota-check/run_a2c_atari.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#SBATCH --job-name=a2c_atari
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/a2c_atari_%j.txt
+#SBATCH --error=slurm_errors/a2c_atari_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="a2c_atari"
+
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/a2c/a2c_atari.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >>> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >>> report.log
+fi
diff --git a/benchmarks/sota-check/run_a2c_mujoco.sh b/benchmarks/sota-check/run_a2c_mujoco.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=a2c_mujoco
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/a2c_mujoco_%j.txt
+#SBATCH --error=slurm_errors/a2c_mujoco_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="a2c_mujoco"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/a2c/a2c_mujoco.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >>> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >>> report.log
+fi
diff --git a/benchmarks/sota-check/run_cql_offline.sh b/benchmarks/sota-check/run_cql_offline.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#SBATCH --job-name=cql_offline
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/cql_offline_%j.txt
+#SBATCH --error=slurm_errors/cql_offline_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="cql_offline"
+
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/cql/cql_offline.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >>> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_cql_online.sh b/benchmarks/sota-check/run_cql_online.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=cql_online
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/cql_online_%j.txt
+#SBATCH --error=slurm_errors/cql_online_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="cql_online"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/cql/cql_online.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_ddpg.sh b/benchmarks/sota-check/run_ddpg.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=ddpg
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/ddpg_%j.txt
+#SBATCH --error=slurm_errors/ddpg_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="ddpg"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/ddpg/ddpg.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_discrete_sac.sh b/benchmarks/sota-check/run_discrete_sac.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=discrete_sac
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/discrete_sac_%j.txt
+#SBATCH --error=slurm_errors/discrete_sac_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="discrete_sac"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/discrete_sac/discrete_sac.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_dqn_atari.sh b/benchmarks/sota-check/run_dqn_atari.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=dqn_atari
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/dqn_atari_%j.txt
+#SBATCH --error=slurm_errors/dqn_atari_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="dqn_atari"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/dqn/dqn_atari.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_dqn_cartpole.sh b/benchmarks/sota-check/run_dqn_cartpole.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=dqn_cartpole
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/dqn_cartpole_%j.txt
+#SBATCH --error=slurm_errors/dqn_cartpole_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="dqn_cartpole"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/dqn/dqn_cartpole.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_dt.sh b/benchmarks/sota-check/run_dt.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=dt
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/dt_offline_%j.txt
+#SBATCH --error=slurm_errors/dt_offline_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="dt_offline"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/decision_transformer/dt.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_dt_online.sh b/benchmarks/sota-check/run_dt_online.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=dt_online
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/dt_online_%j.txt
+#SBATCH --error=slurm_errors/dt_online_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="dt_online"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/decision_transformer/online_dt.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_impala_single_node.sh b/benchmarks/sota-check/run_impala_single_node.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=impala_1node
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/impala_1node_%j.txt
+#SBATCH --error=slurm_errors/impala_1node_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="impala_1node"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/impala/impala_single_node.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_iql_discrete.sh b/benchmarks/sota-check/run_iql_discrete.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=iql_discrete
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/iql_discrete_%j.txt
+#SBATCH --error=slurm_errors/iql_discrete_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="iql_discrete"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/iql/discrete_iql.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >> report.log
+fi
diff --git a/benchmarks/sota-check/run_iql_offline.sh b/benchmarks/sota-check/run_iql_offline.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=iql_offline
+#SBATCH --ntasks=32
+#SBATCH --cpus-per-task=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/iql_offline_%j.txt
+#SBATCH --error=slurm_errors/iql_offline_%j.txt
+
+current_commit=$(git rev-parse --short HEAD)
+project_name="torchrl-example-check-$current_commit"
+group_name="iql_offline"
+export PYTHONPATH=$(dirname $(dirname $PWD))
+python $PYTHONPATH/examples/iql/iql_offline.py \
+  logger.backend=wandb \
+  logger.project_name="$project_name" \
+  logger.group_name="$group_name"
+
+# Capture the exit status of the Python command
+exit_status=$?
+# Write the exit status to a file
+if [ $exit_status -eq 0 ]; then
+  echo "${group_name}_${SLURM_JOB_ID}=success" >>> report.log
+else
+  echo "${group_name}_${SLURM_JOB_ID}=error" >>> report.log
+fi