diff --git a/.github/settings.yml b/.github/settings.yml index 470931e17c9f..440432f59eb4 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -24,6 +24,9 @@ labels: - name: area/indexes color: "#006B75" description: Issues about indexes + - name: area/observability + color: "#E04D33" + description: Issues about metrics, logs, traces, pprof, etc - name: area/oplog color: "#09D521" description: Issues about OpLog @@ -44,6 +47,9 @@ labels: - name: backend/sqlite color: "#2C8FCA" description: Issues about SQLite backend + - name: backend/hana + color: "#2D6AB1" + description: Issues about SAP HANA backend # blog - name: blog/engineering diff --git a/.github/workflows/call-test-integration.yml b/.github/workflows/_integration.yml similarity index 78% rename from .github/workflows/call-test-integration.yml rename to .github/workflows/_integration.yml index 37b12f966d7e..3f6980a9158b 100644 --- a/.github/workflows/call-test-integration.yml +++ b/.github/workflows/_integration.yml @@ -1,4 +1,5 @@ --- +name: Integration # visible at https://github.com/FerretDB/FerretDB/actions on: workflow_call: inputs: @@ -11,6 +12,9 @@ on: shard_total: required: true type: number + coveralls: + required: true + type: boolean disable_filter_pushdown: required: false type: boolean @@ -19,6 +23,13 @@ on: required: false type: boolean default: false + checkout_ref: + required: false + type: string + default: "" + secrets: + FERRETDB_HANA_URL: + required: false env: GOPATH: /home/runner/go @@ -29,13 +40,10 @@ env: GOTOOLCHAIN: local jobs: - test-integration: + run: # make it short to fit in GitHub UI; all parameters are already in the caller's name name: Run - - runs-on: - group: paid - + runs-on: 4-cores timeout-minutes: 20 steps: @@ -43,11 +51,24 @@ jobs: uses: GitHubSecurityLab/actions-permissions/monitor@v1 if: false + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + - name: Checkout code uses: actions/checkout@v3 with: fetch-depth: 0 # for `git describe` to work lfs: false # LFS is used only by website + ref: ${{ inputs.checkout_ref }} + + # for branch.txt on pull_request_target; `main` is already checked out on push / schedule + - name: Name branch + if: github.event_name == 'pull_request_target' + env: + BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + run: git checkout -b $BRANCH - name: Setup Go uses: FerretDB/github-actions/setup-go@main @@ -79,6 +100,7 @@ jobs: ENABLE_SORT_PUSHDOWN=${{ inputs.enable_sort_pushdown }} env: GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }} + FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }} # The token is not required but should make uploads more stable. # If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads. @@ -98,6 +120,7 @@ jobs: verbose: true - name: Upload coverage information to coveralls + if: always() && inputs.coveralls uses: coverallsapp/github-action@v2 with: file: ./integration/integration-${{ inputs.task }}.txt diff --git a/.github/workflows/go-cron.yml b/.github/workflows/go-extra.yml similarity index 77% rename from .github/workflows/go-cron.yml rename to .github/workflows/go-extra.yml index 646d618843cd..529f55a5da2a 100644 --- a/.github/workflows/go-cron.yml +++ b/.github/workflows/go-extra.yml @@ -1,8 +1,10 @@ --- -name: Go Cron +# Integration tests for extra configuration. + +name: Go Extra on: schedule: - - cron: "12 0 * * *" + - cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache env: GOPATH: /home/runner/go @@ -33,10 +35,13 @@ jobs: disable_filter_pushdown: [false, true] enable_sort_pushdown: [false, true] - uses: ./.github/workflows/call-test-integration.yml + # Do not submit to coveralls because it can't handle parallel workflows: + # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515 + uses: ./.github/workflows/_integration.yml with: task: ${{ matrix.task }} shard_index: ${{ matrix.shard_index }} shard_total: ${{ matrix.shard_total }} disable_filter_pushdown: ${{ matrix.disable_filter_pushdown }} enable_sort_pushdown: ${{ matrix.enable_sort_pushdown }} + coveralls: false diff --git a/.github/workflows/go-trust.yml b/.github/workflows/go-trust.yml index 0efa88e774cf..7de592d32eff 100644 --- a/.github/workflows/go-trust.yml +++ b/.github/workflows/go-trust.yml @@ -15,7 +15,7 @@ on: - main - releases/* schedule: - - cron: "12 2 * * *" # after integration workflow to reuse cache + - cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache env: GOPATH: /home/runner/go @@ -27,15 +27,13 @@ env: jobs: integration: - name: Integration ${{ matrix.name }} ${{ matrix.shard_index }}/${{ matrix.shard_total }} - runs-on: - group: paid - timeout-minutes: 20 + # job name must be unique; make it unique and nice + name: ${{ matrix.task }} ${{ matrix.shard_index }}/${{ matrix.shard_total }} # Do not run this job in parallel for any PR change or branch push # to save some resources. concurrency: - group: ${{ github.workflow }}-integration-${{ matrix.name }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }} + group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true if: > @@ -48,87 +46,18 @@ jobs: strategy: fail-fast: false matrix: - include: - - { name: "Hana", task: "hana", shard_index: 1, shard_total: 3 } - - { name: "Hana", task: "hana", shard_index: 2, shard_total: 3 } - - { name: "Hana", task: "hana", shard_index: 3, shard_total: 3 } - - steps: - - name: Setup permissions monitoring - uses: GitHubSecurityLab/actions-permissions/monitor@v1 - if: false - - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 # for `git describe` to work - lfs: false # LFS is used only by website - - - name: Checkout pull request code - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - fetch-depth: 0 - lfs: false - ref: ${{ github.event.pull_request.head.sha }} - - # for branch.txt on pull_request_target; `main` is already checked out on push / schedule - - name: Name branch - if: github.event_name == 'pull_request_target' - env: - BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable - run: git checkout -b $BRANCH - - - name: Setup Go - uses: FerretDB/github-actions/setup-go@main - with: - cache-key: integration - - - name: Install Task - run: go generate -x - working-directory: tools - - - name: Start environment - run: bin/task env-up-detach - - - name: Run init - run: bin/task init - - - name: Wait for and setup environment - run: bin/task env-setup - - - name: Run ${{ matrix.name }} tests (${{ matrix.shard_index }}/${{ matrix.shard_total }}) - run: bin/task test-integration-${{ matrix.task }} SHARD_INDEX=${{ matrix.shard_index }} SHARD_TOTAL=${{ matrix.shard_total }} - env: - GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }} - FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }} - - # The token is not required but should make uploads more stable. - # If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads. - # - # Unfortunately, it seems that tokenless uploads fail too often. - # See https://github.com/codecov/codecov-action/issues/837. - # - # We also can't use ${{ vars.CODECOV_TOKEN }}: https://github.com/orgs/community/discussions/44322 - - name: Upload coverage information to codecov - if: always() - uses: codecov/codecov-action@v3 - with: - token: 22159d7c-856d-4fe9-8fdb-5d9ecff35514 - files: ./integration/integration-${{ matrix.task }}.txt - flags: integration,${{ matrix.task }}-${{ matrix.shard_index }} - fail_ci_if_error: true - verbose: true - - # Do not submit to coveralls because it can't handle parallel workflows: - # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515 - - # we don't want them on CI - - name: Clean test and fuzz caches - if: always() - run: go clean -testcache -fuzzcache - - - name: Check dirty - run: | - git status - git diff --exit-code + task: [hana] + shard_index: [1, 2, 3] + shard_total: [3] + + # Do not submit to coveralls because it can't handle parallel workflows: + # https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515 + uses: ./.github/workflows/_integration.yml + with: + task: ${{ matrix.task }} + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} + checkout_ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.merge_commit_sha || ''}} + coveralls: false + secrets: + FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3efbbc11b648..3fafa6a80e4d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -180,11 +180,12 @@ jobs: include: - { task: "mongodb", shard_index: 1, shard_total: 1 } - uses: ./.github/workflows/call-test-integration.yml + uses: ./.github/workflows/_integration.yml with: task: ${{ matrix.task }} shard_index: ${{ matrix.shard_index }} shard_total: ${{ matrix.shard_total }} + coveralls: true # https://github.com/lemurheavy/coveralls-public/issues/1636 submit-coveralls: diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 51ef095fcc14..1637da1a6e66 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -45,8 +45,7 @@ env: jobs: build: name: Build packages - runs-on: - group: paid + runs-on: 4-cores timeout-minutes: 90 if: > @@ -78,7 +77,7 @@ jobs: with: fetch-depth: 0 lfs: false - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.merge_commit_sha }} # for version.txt on push tags; see https://github.com/actions/checkout/issues/290 - name: Fetch annotated tags diff --git a/.github/workflows/website-preview.yml b/.github/workflows/website-preview.yml index 9a12c01d6cd7..89997b6561b2 100644 --- a/.github/workflows/website-preview.yml +++ b/.github/workflows/website-preview.yml @@ -49,7 +49,7 @@ jobs: with: fetch-depth: 0 # for wrangler to work lfs: true - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.merge_commit_sha }} # for Wrangler / Cloudflare Pages on pull_request_target - name: Name branch @@ -118,7 +118,7 @@ jobs: with: fetch-depth: 0 # for wrangler to work lfs: true - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.merge_commit_sha }} # for Wrangler / Cloudflare Pages on pull_request_target - name: Name branch diff --git a/Taskfile.yml b/Taskfile.yml index 00c671b3ba10..839bd859dfdd 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -187,10 +187,10 @@ tasks: desc: "Run integration tests for several backends in parallel" deps: - test-integration-pg - - test-integration-mongodb - test-integration-sqlite # no test-integration-postgresql yet # no test-integration-hana + - test-integration-mongodb test-integration-pg: desc: "Run integration tests for `pg` handler"