diff --git a/.github/unittest/windows_optdepts/scripts/install.sh b/.github/unittest/windows_optdepts/scripts/install.sh
index b78b2906500..5c425d18a95 100644
--- a/.github/unittest/windows_optdepts/scripts/install.sh
+++ b/.github/unittest/windows_optdepts/scripts/install.sh
@@ -41,15 +41,15 @@ git submodule sync && git submodule update --init --recursive
 printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 if [[ "$TORCH_VERSION" == "nightly" ]]; then
   if $torch_cuda ; then
-    python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu121
+    python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu118
   else
     python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
   fi
 elif [[ "$TORCH_VERSION" == "stable" ]]; then
   if $torch_cuda ; then
-      pip3 install torch --index-url https://download.pytorch.org/whl/cpu
+      python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu118
   else
-      pip3 install torch --index-url https://download.pytorch.org/whl/cu121
+      python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
   fi
 else
   printf "Failed to install pytorch"
diff --git a/.github/workflows/test-windows-optdepts.yml b/.github/workflows/test-windows-optdepts.yml
index 7cd09b9087c..e98b6c1810e 100644
--- a/.github/workflows/test-windows-optdepts.yml
+++ b/.github/workflows/test-windows-optdepts.yml
@@ -55,47 +55,47 @@ jobs:
         ## post_process.sh
         ./.github/unittest/windows_optdepts/scripts/post_process.sh
 
-  unittests-gpu:
-    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
-    with:
-      runner: "windows.g5.4xlarge.nvidia.gpu"
-      repository: pytorch/rl
-      timeout: 40
-      script: |
-        set -euxo pipefail
-
-        export PYTHON_VERSION="3.9"
-        export CUDA_VERSION="11.6"
-        export CU_VERSION="cu116"
-        export torch_cuda="True"
-
-        # TODO: Port this to pytorch/test-infra/.github/workflows/windows_job.yml
-        export PATH="/c/Jenkins/Miniconda3/Scripts:${PATH}"
-
-        echo "PYTHON_VERSION: $PYTHON_VERSION"
-
-        if [[ "${{ github.ref }}" =~ release/* ]]; then
-          export RELEASE=1
-          export TORCH_VERSION=stable
-        else
-          export RELEASE=0
-          export TORCH_VERSION=nightly
-        fi
-
-        ## setup_env.sh
-        ./.github/unittest/windows_optdepts/scripts/setup_env.sh
-
-        ## Install CUDA
-        packaging/windows/internal/cuda_install.bat
-
-        ## Update CUDA Driver
-        packaging/windows/internal/driver_update.bat
-
-        ## install.sh
-        ./.github/unittest/windows_optdepts/scripts/install.sh
-
-        ## run_test.sh
-        ./.github/unittest/windows_optdepts/scripts/run_test.sh
-
-        ## post_process.sh
-        ./.github/unittest/windows_optdepts/scripts/post_process.sh
+#  unittests-gpu:
+#    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+#    with:
+#      runner: "windows.g5.4xlarge.nvidia.gpu"
+#      repository: pytorch/rl
+#      timeout: 40
+#      script: |
+#        set -euxo pipefail
+#
+#        export PYTHON_VERSION="3.9"
+#        export CUDA_VERSION="11.6"
+#        export CU_VERSION="cu116"
+#        export torch_cuda="True"
+#
+#        # TODO: Port this to pytorch/test-infra/.github/workflows/windows_job.yml
+#        export PATH="/c/Jenkins/Miniconda3/Scripts:${PATH}"
+#
+#        echo "PYTHON_VERSION: $PYTHON_VERSION"
+#
+#        if [[ "${{ github.ref }}" =~ release/* ]]; then
+#          export RELEASE=1
+#          export TORCH_VERSION=stable
+#        else
+#          export RELEASE=0
+#          export TORCH_VERSION=nightly
+#        fi
+#
+#        ## setup_env.sh
+#        ./.github/unittest/windows_optdepts/scripts/setup_env.sh
+#
+#        ## Install CUDA
+#        packaging/windows/internal/cuda_install.bat
+#
+#        ## Update CUDA Driver
+#        packaging/windows/internal/driver_update.bat
+#
+#        ## install.sh
+#        ./.github/unittest/windows_optdepts/scripts/install.sh
+#
+#        ## run_test.sh
+#        ./.github/unittest/windows_optdepts/scripts/run_test.sh
+#
+#        ## post_process.sh
+#        ./.github/unittest/windows_optdepts/scripts/post_process.sh
diff --git a/test/test_env.py b/test/test_env.py
index 6ff97e0c37f..a30603aa803 100644
--- a/test/test_env.py
+++ b/test/test_env.py
@@ -2622,8 +2622,11 @@ def test_parallel_another_ctx():
         assert env.rollout(3) is not None
         assert env._workers[0]._start_method == other_sm
     finally:
-        env.close()
-        del env
+        try:
+            env.close()
+            del env
+        except RuntimeError:
+            pass
 
 
 if __name__ == "__main__":