Skip to content

Commit

Permalink
Merge branch 'main' into client_metadata_not_mutated
Browse files Browse the repository at this point in the history
  • Loading branch information
kropidlowsky authored Sep 24, 2023
2 parents 436ba21 + dabbeb3 commit 0348c59
Show file tree
Hide file tree
Showing 49 changed files with 1,905 additions and 406 deletions.
9 changes: 8 additions & 1 deletion .github/settings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@ labels:
description: Issues about OpLog
- name: area/params
color: "#FBCA04"
description: Issues about parameters extraction
description: Issues about command parameters extraction
- name: area/pushdown
color: "#83F7BA"
description: Issues about query pushdown
- name: area/types
color: "#0354F4"
description: Issues about data types, BSON, etc

# backend
- name: backend/pg
Expand Down Expand Up @@ -93,6 +96,10 @@ labels:
color: "#5319E7"
description: Good issues for new external contributors

- name: hacktoberfest
color: "#FEFBA2"
description: Good issues for Hacktoberfest

- name: help wanted
color: "#006B75"
description: Issues that require extra attention
Expand Down
115 changes: 115 additions & 0 deletions .github/workflows/call-test-integration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
---
on:
workflow_call:
inputs:
task:
required: true
type: string
shard_index:
required: true
type: number
shard_total:
required: true
type: number
disable_filter_pushdown:
required: false
type: boolean
default: false
enable_sort_pushdown:
required: false
type: boolean
default: false

env:
GOPATH: /home/runner/go
GOCACHE: /home/runner/go/cache
GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint
GOMODCACHE: /home/runner/go/mod
GOPROXY: https://proxy.golang.org
GOTOOLCHAIN: local

jobs:
test-integration:
# make it short to fit in GitHub UI; all parameters are already in the caller's name
name: Run

runs-on:
group: paid

timeout-minutes: 20

steps:
- name: Setup permissions monitoring
uses: GitHubSecurityLab/actions-permissions/monitor@v1
if: false

- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website

- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
cache-key: integration

- name: Install Task
run: go generate -x
working-directory: tools

- name: Start environment
run: bin/task env-up-detach

- name: Run init
run: bin/task init

- name: Wait for and setup environment
run: bin/task env-setup

- name: >
Run ${{ inputs.task }} tests
(${{ inputs.shard_index }}/${{ inputs.shard_total }},
filter=${{ !inputs.disable_filter_pushdown }}, sort=${{ inputs.enable_sort_pushdown }})
run: >
bin/task test-integration-${{ inputs.task }}
SHARD_INDEX=${{ inputs.shard_index }}
SHARD_TOTAL=${{ inputs.shard_total }}
DISABLE_FILTER_PUSHDOWN=${{ inputs.disable_filter_pushdown }}
ENABLE_SORT_PUSHDOWN=${{ inputs.enable_sort_pushdown }}
env:
GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }}
# The token is not required but should make uploads more stable.
# If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads.
#
# Unfortunately, it seems that tokenless uploads fail too often.
# See https://github.com/codecov/codecov-action/issues/837.
#
# We also can't use ${{ vars.CODECOV_TOKEN }}: https://github.com/orgs/community/discussions/44322
- name: Upload coverage information to codecov
if: always()
uses: codecov/codecov-action@v3
with:
token: 22159d7c-856d-4fe9-8fdb-5d9ecff35514
files: ./integration/integration-${{ inputs.task }}.txt
flags: integration,${{ inputs.task }}-${{ inputs.shard_index }},filter-${{ !inputs.disable_filter_pushdown }},sort-${{ inputs.enable_sort_pushdown }}
fail_ci_if_error: true
verbose: true

- name: Upload coverage information to coveralls
uses: coverallsapp/github-action@v2
with:
file: ./integration/integration-${{ inputs.task }}.txt
flag-name: integration-${{ inputs.task }}-${{ inputs.shard_index }}-filter-${{ !inputs.disable_filter_pushdown }}-sort-${{ inputs.enable_sort_pushdown }}
parallel: true

# we don't want them on CI
- name: Clean test and fuzz caches
if: always()
run: go clean -testcache -fuzzcache

- name: Check dirty
run: |
git status
git diff --exit-code
42 changes: 42 additions & 0 deletions .github/workflows/go-cron.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
---
name: Go Cron
on:
schedule:
- cron: "12 0 * * *"

env:
GOPATH: /home/runner/go
GOCACHE: /home/runner/go/cache
GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint
GOMODCACHE: /home/runner/go/mod
GOPROXY: https://proxy.golang.org
GOTOOLCHAIN: local

jobs:
integration:
# job name must be unique; make it unique and nice
name: >
${{ matrix.task }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
(filter=${{ matrix.disable_filter_pushdown }}, sort=${{ matrix.enable_sort_pushdown }})
# To avoid conflict with go.yml.
concurrency:
group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.shard_index }}-${{ matrix.disable_filter_pushdown }}-${{ matrix.enable_sort_pushdown }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true

strategy:
fail-fast: false
matrix:
task: [pg, postgresql, sqlite]
shard_index: [1, 2, 3]
shard_total: [3]
disable_filter_pushdown: [false, true]
enable_sort_pushdown: [false, true]

uses: ./.github/workflows/call-test-integration.yml
with:
task: ${{ matrix.task }}
shard_index: ${{ matrix.shard_index }}
shard_total: ${{ matrix.shard_total }}
disable_filter_pushdown: ${{ matrix.disable_filter_pushdown }}
enable_sort_pushdown: ${{ matrix.enable_sort_pushdown }}
91 changes: 12 additions & 79 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,98 +160,31 @@ jobs:
git diff --exit-code
integration:
name: Integration ${{ matrix.name }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
runs-on:
group: paid
timeout-minutes: 20
# job name must be unique; make it unique and nice
name: ${{ matrix.task }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}

# Do not run this job in parallel for any PR change or branch push
# to save some resources.
concurrency:
group: ${{ github.workflow }}-integration-${{ matrix.name }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }}
group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true

if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')

strategy:
fail-fast: false
matrix:
task: [pg, postgresql, sqlite]
shard_index: [1, 2, 3]
shard_total: [3]
include:
- { name: "MongoDB", task: "mongodb", shard_index: 1, shard_total: 1 }
- { name: "PostgreSQL", task: "pg", shard_index: 1, shard_total: 3 }
- { name: "PostgreSQL", task: "pg", shard_index: 2, shard_total: 3 }
- { name: "PostgreSQL", task: "pg", shard_index: 3, shard_total: 3 }
- { name: "SQLite", task: "sqlite", shard_index: 1, shard_total: 3 }
- { name: "SQLite", task: "sqlite", shard_index: 2, shard_total: 3 }
- { name: "SQLite", task: "sqlite", shard_index: 3, shard_total: 3 }
- { task: "mongodb", shard_index: 1, shard_total: 1 }

steps:
- name: Setup permissions monitoring
uses: GitHubSecurityLab/actions-permissions/monitor@v1
if: false

- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website

- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
cache-key: integration

- name: Install Task
run: go generate -x
working-directory: tools

- name: Start environment
run: bin/task env-up-detach

- name: Run init
run: bin/task init

- name: Wait for and setup environment
run: bin/task env-setup

- name: Run ${{ matrix.name }} tests (${{ matrix.shard_index }}/${{ matrix.shard_total }})
run: bin/task test-integration-${{ matrix.task }} SHARD_INDEX=${{ matrix.shard_index }} SHARD_TOTAL=${{ matrix.shard_total }}
env:
GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }}

# The token is not required but should make uploads more stable.
# If secrets are unavailable (for example, for a pull request from a fork), it fallbacks to the tokenless uploads.
#
# Unfortunately, it seems that tokenless uploads fail too often.
# See https://github.com/codecov/codecov-action/issues/837.
#
# We also can't use ${{ vars.CODECOV_TOKEN }}: https://github.com/orgs/community/discussions/44322
- name: Upload coverage information to codecov
if: always()
uses: codecov/codecov-action@v3
with:
token: 22159d7c-856d-4fe9-8fdb-5d9ecff35514
files: ./integration/integration-${{ matrix.task }}.txt
flags: integration,${{ matrix.task }}-${{ matrix.shard_index }}
fail_ci_if_error: true
verbose: true

- name: Upload coverage information to coveralls
uses: coverallsapp/github-action@v2
with:
file: ./integration/integration-${{ matrix.task }}.txt
flag-name: integration-${{ matrix.task }}-${{ matrix.shard_index }}
parallel: true

# we don't want them on CI
- name: Clean test and fuzz caches
if: always()
run: go clean -testcache -fuzzcache

- name: Check dirty
run: |
git status
git diff --exit-code
uses: ./.github/workflows/call-test-integration.yml
with:
task: ${{ matrix.task }}
shard_index: ${{ matrix.shard_index }}
shard_total: ${{ matrix.shard_total }}

# https://github.com/lemurheavy/coveralls-public/issues/1636
submit-coveralls:
Expand Down
2 changes: 2 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ linters-settings:
files:
- $all
- "!**/internal/backends/sqlite/*.go"
- "!**/internal/backends/postgresql/*.go"
- "!**/internal/backends/postgresql/metadata/*.go"
- "!**/internal/handlers/pg/pgdb/*.go"
- "!**/internal/handlers/hana/hanadb/*.go"
deny:
Expand Down
29 changes: 27 additions & 2 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ env:
vars:
SHARD_INDEX: 1
SHARD_TOTAL: 1
DISABLE_FILTER_PUSHDOWN: false
ENABLE_SORT_PUSHDOWN: false
TEST_RUN: ""
TEST_TIMEOUT: 35m
BENCH_TIME: 5s
Expand Down Expand Up @@ -187,6 +189,7 @@ tasks:
- test-integration-pg
- test-integration-mongodb
- test-integration-sqlite
# no test-integration-postgresql yet
# no test-integration-hana

test-integration-pg:
Expand All @@ -200,12 +203,32 @@ tasks:
-target-tls
-postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?pool_max_conns=50&search_path='
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem'
-disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}}
-enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}}
vars:
SHARD_RUN:
sh: go run -C .. ./cmd/envtool tests shard --index={{.SHARD_INDEX}} --total={{.SHARD_TOTAL}}

test-integration-postgresql:
desc: "Run integration tests for new `postgresql` backend"
dir: integration
cmds:
- >
go test -count=1 -run='{{or .TEST_RUN .SHARD_RUN}}' -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverpkg=../...
-coverprofile=integration-postgresql.txt
-target-backend=ferretdb-pg
-target-tls
-postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?pool_max_conns=50&search_path='
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem'
-use-new-pg
-disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}}
-enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}}
vars:
SHARD_RUN:
sh: go run -C .. ./cmd/envtool tests shard --index={{.SHARD_INDEX}} --total={{.SHARD_TOTAL}}

test-integration-sqlite:
desc: "Run integration tests for `sqlite` handler"
desc: "Run integration tests for `sqlite` backend"
dir: integration
cmds:
- >
Expand All @@ -215,6 +238,8 @@ tasks:
-sqlite-url=file:../tmp/sqlite-tests/
-target-tls
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem'
-disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}}
-enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}}
vars:
SHARD_RUN:
sh: go run -C .. ./cmd/envtool tests shard --index={{.SHARD_INDEX}} --total={{.SHARD_TOTAL}}
Expand Down Expand Up @@ -569,7 +594,7 @@ tasks:
fmt-yaml:
desc: "Format YAML files"
cmds:
- docker compose run --rm prettier --write --parser=yaml "**/*.yml"
- docker compose run --rm prettier --write --parser=yaml --print-width=120 "**/*.yml"

docs:
desc: "Format, lint and build documentation"
Expand Down
Loading

0 comments on commit 0348c59

Please sign in to comment.