Refactor template rendering and automated issue creation (#113)

pmeier · Jan 18, 2023 · f9fb663 · f9fb663
1 parent 853b505
commit f9fb663
Show file tree

Hide file tree

Showing 12 changed files with 135 additions and 101 deletions.
diff --git a/.github/actions/publish-from-template/action.yml b/.github/actions/publish-from-template/action.yml
@@ -0,0 +1,25 @@
+name: publish-from-template
+description: "Publish information from a template"
+
+inputs:
+  filename:
+    description: Path to issue template. Usually in .github/issue-templates
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: Render template
+      shell: bash
+      env: ${{ env }}
+      run:
+        python .github/actions/publish-from-template/render_template.py ${{
+        inputs.filename }}
+
+    - uses: JasonEtco/create-an-issue@v2.6.0
+      if: github.event_name == 'schedule'
+      env: ${{ env }}
+      with:
+        filename: ${{ inputs.filename }}
+        update_existing: false
diff --git a/.github/actions/publish-from-template/render_template.py b/.github/actions/publish-from-template/render_template.py
@@ -0,0 +1,17 @@
+import os
+import pathlib
+import sys
+
+import jinja2
+
+
+def main(template_path):
+    loader = jinja2.FileSystemLoader(searchpath=template_path.parent)
+    env = jinja2.Environment(loader=loader)
+    template = env.get_template(template_path.name)
+    print(template.render(env=os.environ))
+
+
+if __name__ == "__main__":
+    template_path = pathlib.Path(sys.argv[1])
+    main(template_path)
diff --git a/.github/incorrect_pytorch_dists_issue_template.md b/.github/incorrect_pytorch_dists_issue_template.md
diff --git a/.github/issue-templates/packages-out-of-sync.md b/.github/issue-templates/packages-out-of-sync.md
@@ -0,0 +1,16 @@
+---
+title: `light-the-torch` is out of sync with the PyTorch package indices
+assignees: pmeier
+---
+{%- if env.MISSING|length %}
+The following packages are available, but not patched:
+{% for package in env.MISSING.split(",") %}
+  - `{{ package }}`
+{%- endfor %}
+{%- endif %}
+{% if env.EXTRA|length %}
+The following packages are patched, but not available:
+{% for package in env.EXTRA.split(",") %}
+  - `{{ package }}`
+{%- endfor %}
+{%- endif %}
diff --git a/.github/pip_latest_failure_issue_template.md → ...hub/issue-templates/pip-latest-failure.md b/.github/pip_latest_failure_issue_template.md → ...hub/issue-templates/pip-latest-failure.md
diff --git a/.github/pip_latest_success_issue_template.md → ...hub/issue-templates/pip-latest-success.md b/.github/pip_latest_success_issue_template.md → ...hub/issue-templates/pip-latest-success.md
diff --git a/.github/workflows/check-available-pytorch-dists.yml b/.github/workflows/check-available-pytorch-dists.yml
diff --git a/.github/workflows/check-pytorch-package-indices.yml b/.github/workflows/check-pytorch-package-indices.yml
@@ -0,0 +1,47 @@
+name: check-available-pytorch-dists
+
+on:
+  pull_request:
+    paths:
+      - .github/issue-templates/packages-out-of-sync.md
+      - .github/workflows/check-available-pytorch-dists.yml
+      - scripts/check_pytorch_package_indices.py
+
+  schedule:
+    - cron: "0 4 * * *"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup development environment
+        uses: ./.github/actions/setup-dev-env
+
+      - name: Check available packages on PyTorch indices
+        id: packages
+        run: |
+          OUT=$(python scripts/check_pytorch_package_indices.py)
+
+          MISSING=$(echo $OUT | jq -r '.missing | join(",")')
+          echo "missing=${MISSING}" >> $GITHUB_OUTPUT
+
+          EXTRA=$(echo $OUT | jq -r '.extra | join(",")')
+          echo "extra=${EXTRA}" >> $GITHUB_OUTPUT
+
+          [ -z "${MISSING}${EXTRA}" ];
+
+      - if: failure()
+        name: Publish information from template
+        uses: ./.github/actions/publish-from-template
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MISSING: ${{ steps.packages.outputs.missing }}
+          EXTRA: ${{ steps.packages.outputs.extra }}
+        with:
+          filename: .github/issue-templates/packages-out-of-sync.md
diff --git a/.github/workflows/tests-pip-latest.yml b/.github/workflows/tests-pip-latest.yml
@@ -3,7 +3,9 @@ name: tests-pip-latest
 on:
   pull_request:
     paths:
-      - ".github/workflows/tests-pip-latest.yml"
+      - .github/issue-templates/pip-latest-failure.md
+      - .github/issue-templates/pip-latest-success.md
+      - .github/workflows/tests-pip-latest.yml
 
   schedule:
     - cron: "0 4 * * *"
@@ -52,28 +54,24 @@ jobs:
         if: steps.current.outputs.version != steps.latest.outputs.version
         run: doit test
 
-      - uses: JasonEtco/create-an-issue@v2.6.0
-        if:
-          failure() && github.event_name == 'schedule' && steps.tests.outcome ==
-          'failure'
+      - if: failure() && steps.tests.outcome == 'failure'
+        name: Publish information from template
+        uses: ./.github/actions/publish-from-template
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
           ID: ${{ github.run_id }}
           VERSION: ${{ steps.latest.outputs.version }}
         with:
-          filename: .github/pip_latest_failure_issue_template.md
-          update_existing: false
+          filename: .github/issue-templates/pip-latest-failure.md
 
-      - uses: JasonEtco/create-an-issue@v2.6.0
-        if:
-          github.event_name == 'schedule' && steps.tests.outcome == 'success' &&
-          matrix.release == 'stable'
+      - if: steps.tests.outcome == 'success' && matrix.release == 'stable'
+        name: Publish information from template
+        uses: ./.github/actions/publish-from-template
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
           ID: ${{ github.run_id }}
           VERSION: ${{ steps.latest.outputs.version }}
         with:
-          filename: .github/pip_latest_success_issue_template.md
-          update_existing: false
+          filename: .github/issue-templates/pip-latest-success.md
diff --git a/.prettierignore b/.prettierignore
@@ -0,0 +1 @@
+.github/issue-templates
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -14,3 +14,4 @@ check-wheel-contents
 requests
 beautifulsoup4
 jinja2
+tqdm
diff --git a/scripts/check_available_pytorch_dists.py → scripts/check_pytorch_package_indices.py b/scripts/check_available_pytorch_dists.py → scripts/check_pytorch_package_indices.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python
-
 import itertools
+import json
 
 import requests
+import tqdm
 from bs4 import BeautifulSoup
 
 from light_the_torch._cb import _MINIMUM_DRIVER_VERSIONS, CPUBackend, CUDABackend
@@ -33,16 +33,19 @@
 }
 COMPUTATION_BACKENDS.add(CPUBackend())
 
-EXTRA_INDEX_URLS = set(
-    itertools.chain.from_iterable(
-        get_extra_index_urls(COMPUTATION_BACKENDS, channel) for channel in iter(Channel)
+EXTRA_INDEX_URLS = sorted(
+    set(
+        itertools.chain.from_iterable(
+            get_extra_index_urls(COMPUTATION_BACKENDS, channel)
+            for channel in iter(Channel)
+        )
     )
 )
 
 
 def main():
     available = set()
-    for url in EXTRA_INDEX_URLS:
+    for url in tqdm.tqdm(EXTRA_INDEX_URLS):
         response = requests.get(url)
         if not response.ok:
             continue
@@ -52,11 +55,14 @@ def main():
         available.update(tag.string for tag in soup.find_all(name="a"))
     available = available - EXCLUDED_PYTORCH_DIST
 
-    missing = available - PATCHED_PYTORCH_DISTS
-    extra = PATCHED_PYTORCH_DISTS - available
-
-    if missing or extra:
-        print(",".join(sorted(available)))
+    print(
+        json.dumps(
+            dict(
+                missing=sorted(available - PATCHED_PYTORCH_DISTS),
+                extra=sorted(PATCHED_PYTORCH_DISTS - available),
+            )
+        )
+    )
 
 
 if __name__ == "__main__":
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,3 +14,4 @@ check-wheel-contents @@
     requests
     beautifulsoup4
     jinja2
+    tqdm