Fix daily link checker (github#28346)

* reduce linkinator concurrency from 300 to 100 * reduce it and use self-hosted * try again * actually do concurrency of 10 * move the override to the Action workflow * Trying `DISABLE_RENDERING_CACHE` * back to 100? * super low concurrency * 3?? * 1?? * try self-hosted + 100 concurrency Co-authored-by: Peter Bengtsson <mail@peterbe.com>
adisu2022 · Jun 9, 2022 · b3af722 · b3af722
1 parent 831a93b
commit b3af722
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 4 deletions.
diff --git a/.github/workflows/check-all-english-links.yml b/.github/workflows/check-all-english-links.yml
@@ -17,7 +17,7 @@ jobs:
   check_all_english_links:
     name: Check all links
     if: github.repository == 'github/docs-internal'
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
     env:
       GITHUB_TOKEN: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
       FIRST_RESPONDER_PROJECT: Docs content first responder
@@ -56,10 +56,21 @@ jobs:
           # The default is 10s. But because this runs overnight, we can
           # be a lot more patient.
           REQUEST_TIMEOUT: 20000
+          # The default is 300 which works OK on a fast macbook pro
+          # but so well in Actions.
+          LINKINATOR_CONCURRENCY: 100
         run: |
-          node server.mjs &
+          node server.mjs > /tmp/stdout.log 2> /tmp/stderr.log &
           sleep 5
-          curl --retry-connrefused --retry 3 -I http://localhost:4000/
+          curl --retry-connrefused --retry 4 -I http://localhost:4000/
+
+      - if: ${{ failure() }}
+        name: Debug server outputs on errors
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log
 
       - name: Run script
         run: |

diff --git a/script/check-english-links.js b/script/check-english-links.js
@@ -43,6 +43,8 @@ const DISPLAY_MAX_LENGTH = parseInt(process.env.DISPLAY_MAX_LENGTH || '30000', 1
 // Links with these codes may or may not really be broken.
 const retryStatusCodes = [429, 503, 'Invalid']
 
+const LINKINATOR_CONCURRENCY = parseInt(process.env.LINKINATOR_CONCURRENCY || '300')
+
 program
   .description('Check all links in the English docs.')
   .option(
@@ -73,7 +75,7 @@ const enterpriseReleasesToSkip = new RegExp(`${root}.+?[/@](${deprecated.join('|
 
 const config = {
   path: program.opts().path || englishRoot,
-  concurrency: 300,
+  concurrency: LINKINATOR_CONCURRENCY,
   // If this is a dry run, turn off recursion.
   recurse: !program.opts().dryRun,
   silent: true,