FerretDB · AlekSi · Sep 25, 2023 · Sep 24, 2023 · Sep 24, 2023 · Sep 25, 2023
@@ -24,6 +24,9 @@ labels:
   - name: area/indexes
     color: "#006B75"
     description: Issues about indexes
+  - name: area/observability
+    color: "#E04D33"
+    description: Issues about metrics, logs, traces, pprof, etc
   - name: area/oplog
     color: "#09D521"
     description: Issues about OpLog
@@ -44,6 +47,9 @@ labels:
   - name: backend/sqlite
     color: "#2C8FCA"
     description: Issues about SQLite backend
+  - name: backend/hana
+    color: "#2D6AB1"
+    description: Issues about SAP HANA backend
 
   # blog
   - name: blog/engineering

@@ -1,4 +1,5 @@
 ---
+name: Integration # visible at https://github.com/FerretDB/FerretDB/actions
 on:
   workflow_call:
     inputs:
@@ -11,6 +12,9 @@ on:
       shard_total:
         required: true
         type: number
+      coveralls:
+        required: true
+        type: boolean
       disable_filter_pushdown:
         required: false
         type: boolean
@@ -19,6 +23,13 @@ on:
         required: false
         type: boolean
         default: false
+      checkout_ref:
+        required: false
+        type: string
+        default: ""
+    secrets:
+      FERRETDB_HANA_URL:
+        required: false
 
 env:
   GOPATH: /home/runner/go
@@ -29,25 +40,35 @@ env:
   GOTOOLCHAIN: local
 
 jobs:
-  test-integration:
+  run:
     # make it short to fit in GitHub UI; all parameters are already in the caller's name
     name: Run
-
-    runs-on:
-      group: paid
-
+    runs-on: 4-cores
     timeout-minutes: 20
 
     steps:
       - name: Setup permissions monitoring
         uses: GitHubSecurityLab/actions-permissions/monitor@v1
         if: false
 
+      - name: Dump github context
+        run: echo "$GITHUB_CONTEXT"
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github) }}
+
       - name: Checkout code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0 # for `git describe` to work
           lfs: false # LFS is used only by website
+          ref: ${{ inputs.checkout_ref }}
+
+      # for branch.txt on pull_request_target; `main` is already checked out on push / schedule
+      - name: Name branch
+        if: github.event_name == 'pull_request_target'
+        env:
+          BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+        run: git checkout -b $BRANCH
 
       - name: Setup Go
         uses: FerretDB/github-actions/setup-go@main
@@ -79,6 +100,7 @@ jobs:
           ENABLE_SORT_PUSHDOWN=${{ inputs.enable_sort_pushdown }}
         env:
           GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }}
+          FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }}
 
       # The token is not required but should make uploads more stable.
       # If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads.
@@ -98,6 +120,7 @@ jobs:
           verbose: true
 
       - name: Upload coverage information to coveralls
+        if: always() && inputs.coveralls
         uses: coverallsapp/github-action@v2
         with:
           file: ./integration/integration-${{ inputs.task }}.txt

@@ -1,8 +1,10 @@
 ---
-name: Go Cron
+# Integration tests for extra configuration.
+
+name: Go Extra
 on:
   schedule:
-    - cron: "12 0 * * *"
+    - cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache
 
 env:
   GOPATH: /home/runner/go
@@ -33,10 +35,13 @@ jobs:
         disable_filter_pushdown: [false, true]
         enable_sort_pushdown: [false, true]
 
-    uses: ./.github/workflows/call-test-integration.yml
+    # Do not submit to coveralls because it can't handle parallel workflows:
+    # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515
+    uses: ./.github/workflows/_integration.yml
     with:
       task: ${{ matrix.task }}
       shard_index: ${{ matrix.shard_index }}
       shard_total: ${{ matrix.shard_total }}
       disable_filter_pushdown: ${{ matrix.disable_filter_pushdown }}
       enable_sort_pushdown: ${{ matrix.enable_sort_pushdown }}
+      coveralls: false
@@ -15,7 +15,7 @@ on:
       - main
       - releases/*
   schedule:
-    - cron: "12 2 * * *" # after integration workflow to reuse cache
+    - cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache
 
 env:
   GOPATH: /home/runner/go
@@ -27,15 +27,13 @@ env:
 
 jobs:
   integration:
-    name: Integration ${{ matrix.name }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
-    runs-on:
-      group: paid
-    timeout-minutes: 20
+    # job name must be unique; make it unique and nice
+    name: ${{ matrix.task }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
 
     # Do not run this job in parallel for any PR change or branch push
     # to save some resources.
     concurrency:
-      group: ${{ github.workflow }}-integration-${{ matrix.name }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }}
+      group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: true
 
     if: >
@@ -48,87 +46,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - { name: "Hana", task: "hana", shard_index: 1, shard_total: 3 }
-          - { name: "Hana", task: "hana", shard_index: 2, shard_total: 3 }
-          - { name: "Hana", task: "hana", shard_index: 3, shard_total: 3 }
-
-    steps:
-      - name: Setup permissions monitoring
-        uses: GitHubSecurityLab/actions-permissions/monitor@v1
-        if: false
-
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # for `git describe` to work
-          lfs: false # LFS is used only by website
-
-      - name: Checkout pull request code
-        if: github.event_name == 'pull_request_target'
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: false
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      # for branch.txt on pull_request_target; `main` is already checked out on push / schedule
-      - name: Name branch
-        if: github.event_name == 'pull_request_target'
-        env:
-          BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-        run: git checkout -b $BRANCH
-
-      - name: Setup Go
-        uses: FerretDB/github-actions/setup-go@main
-        with:
-          cache-key: integration
-
-      - name: Install Task
-        run: go generate -x
-        working-directory: tools
-
-      - name: Start environment
-        run: bin/task env-up-detach
-
-      - name: Run init
-        run: bin/task init
-
-      - name: Wait for and setup environment
-        run: bin/task env-setup
-
-      - name: Run ${{ matrix.name }} tests (${{ matrix.shard_index }}/${{ matrix.shard_total }})
-        run: bin/task test-integration-${{ matrix.task }} SHARD_INDEX=${{ matrix.shard_index }} SHARD_TOTAL=${{ matrix.shard_total }}
-        env:
-          GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }}
-          FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }}
-
-      # The token is not required but should make uploads more stable.
-      # If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads.
-      #
-      # Unfortunately, it seems that tokenless uploads fail too often.
-      # See https://github.com/codecov/codecov-action/issues/837.
-      #
-      # We also can't use ${{ vars.CODECOV_TOKEN }}: https://github.com/orgs/community/discussions/44322
-      - name: Upload coverage information to codecov
-        if: always()
-        uses: codecov/codecov-action@v3
-        with:
-          token: 22159d7c-856d-4fe9-8fdb-5d9ecff35514
-          files: ./integration/integration-${{ matrix.task }}.txt
-          flags: integration,${{ matrix.task }}-${{ matrix.shard_index }}
-          fail_ci_if_error: true
-          verbose: true
-
-      # Do not submit to coveralls because it can't handle parallel workflows:
-      # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515
-
-      # we don't want them on CI
-      - name: Clean test and fuzz caches
-        if: always()
-        run: go clean -testcache -fuzzcache
-
-      - name: Check dirty
-        run: |
-          git status
-          git diff --exit-code
+        task: [hana]
+        shard_index: [1, 2, 3]
+        shard_total: [3]
+
+    # Do not submit to coveralls because it can't handle parallel workflows:
+    # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515
+    uses: ./.github/workflows/_integration.yml
+    with:
+      task: ${{ matrix.task }}
+      shard_index: ${{ matrix.shard_index }}
+      shard_total: ${{ matrix.shard_total }}
+      checkout_ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || ''}}
+      coveralls: false
+    secrets:
+      FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }}
@@ -180,11 +180,12 @@ jobs:
         include:
           - { task: "mongodb", shard_index: 1, shard_total: 1 }
 
-    uses: ./.github/workflows/call-test-integration.yml
+    uses: ./.github/workflows/_integration.yml
     with:
       task: ${{ matrix.task }}
       shard_index: ${{ matrix.shard_index }}
       shard_total: ${{ matrix.shard_total }}
+      coveralls: true
 
   # https://github.com/lemurheavy/coveralls-public/issues/1636
   submit-coveralls:

@@ -45,8 +45,7 @@ env:
 jobs:
   build:
     name: Build packages
-    runs-on:
-      group: paid
+    runs-on: 4-cores
     timeout-minutes: 90
 
     if: >
@@ -78,7 +77,7 @@ jobs:
         with:
           fetch-depth: 0
           lfs: false
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
 
       # for version.txt on push tags; see https://github.com/actions/checkout/issues/290
       - name: Fetch annotated tags

@@ -49,7 +49,7 @@ jobs:
         with:
           fetch-depth: 0 # for wrangler to work
           lfs: true
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
 
       # for Wrangler / Cloudflare Pages on pull_request_target
       - name: Name branch
@@ -118,7 +118,7 @@ jobs:
         with:
           fetch-depth: 0 # for wrangler to work
           lfs: true
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
 
       # for Wrangler / Cloudflare Pages on pull_request_target
       - name: Name branch

@@ -187,10 +187,10 @@ tasks:
     desc: "Run integration tests for several backends in parallel"
     deps:
       - test-integration-pg
-      - test-integration-mongodb
       - test-integration-sqlite
       # no test-integration-postgresql yet
       # no test-integration-hana
+      - test-integration-mongodb
 
   test-integration-pg:
     desc: "Run integration tests for `pg` handler"