diff --git a/.github/scripts/wait_for_ssh_to_drain.sh b/.github/scripts/wait_for_ssh_to_drain.sh new file mode 100755 index 0000000000000..f33d807640332 --- /dev/null +++ b/.github/scripts/wait_for_ssh_to_drain.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eou pipefail + +echo "Holding runner for 2 hours until all ssh sessions have logged out" +for _ in $(seq 1440); do + # Break if no ssh session exists anymore + if [ "$(who)" = "" ]; then + break + fi + echo "." + sleep 5 +done diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index a1ca02cda617f..02c2f2df9dfb1 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -82,6 +82,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -153,6 +154,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -231,6 +237,10 @@ jobs: path: artifacts.zip {%- endif %} + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -294,6 +304,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -400,6 +415,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -496,6 +515,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enables SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -564,6 +588,10 @@ jobs: name: pytorch_github_io if-no-files-found: error path: pytorch_github_io.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml index cf0c35a76d0ee..feae626796856 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml @@ -49,6 +49,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -120,6 +121,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -174,6 +180,10 @@ jobs: run: | # Ensure the working directory gets chowned back to the current user docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml index 5b9d29a523beb..3dd00b209b79d 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml @@ -49,6 +49,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -120,6 +121,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -174,6 +180,10 @@ jobs: run: | # Ensure the working directory gets chowned back to the current user docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml b/.github/workflows/generated-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml index 6783938cbbed6..03e95c92d6691 100644 --- a/.github/workflows/generated-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml +++ b/.github/workflows/generated-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml @@ -49,6 +49,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -120,6 +121,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -196,6 +202,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -254,6 +264,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -360,6 +375,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index 21076f66337e6..04a7106ce7321 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -58,6 +58,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -129,6 +130,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -205,6 +211,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -264,6 +274,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -370,6 +385,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml index ed83d5d4a8d13..fedee4ffcb48f 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml @@ -58,6 +58,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -129,6 +130,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -205,6 +211,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -264,6 +274,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -370,6 +385,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml index 3567edc1648cc..4e3db7fe7bfb7 100644 --- a/.github/workflows/generated-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml @@ -49,6 +49,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -120,6 +121,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -196,6 +202,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -254,6 +264,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -360,6 +375,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index 539131b37780a..e6305961a3303 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -50,6 +50,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -121,6 +122,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -197,6 +203,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -255,6 +265,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -361,6 +376,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -452,6 +471,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enables SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -520,6 +544,10 @@ jobs: name: pytorch_github_io if-no-files-found: error path: pytorch_github_io.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index 9d140f10a65b9..19b9beff46101 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -49,6 +49,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: diff --git a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml index e8f203361d73c..f0d780224826f 100644 --- a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml @@ -56,6 +56,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -127,6 +128,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -181,6 +187,10 @@ jobs: run: | # Ensure the working directory gets chowned back to the current user docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml index e734396ab6d2b..6d408eec952b1 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7.yml @@ -56,6 +56,7 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -127,6 +128,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -203,6 +209,10 @@ jobs: if-no-files-found: error path: artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: | @@ -262,6 +272,11 @@ jobs: - name: Clean workspace run: | rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout PyTorch uses: actions/checkout@v2 with: @@ -368,6 +383,10 @@ jobs: if-no-files-found: error path: test-reports-*.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh - name: Clean up docker images if: always() run: |