Merge branch 'main' into add-more-tests-2283

FerretDB · Dec 21, 2023 · 05009bd · 05009bd
2 parents 75426f9 + c5e44a7
commit 05009bd
Show file tree

Hide file tree

Showing 21 changed files with 542 additions and 36 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -125,7 +125,7 @@ jobs:
 
       - name: Deploy to GitHub Pages
         id: deploy
-        uses: actions/deploy-pages@v3
+        uses: actions/deploy-pages@v4
         with:
           artifact_name: docs
           timeout: 300000 # 5*60*1000 milliseconds
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
@@ -95,7 +95,6 @@ jobs:
         run: git checkout -b $BRANCH
 
       - name: Setup Go
-        id: setup
         uses: FerretDB/github-actions/setup-go@main
 
       - name: Install Task

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -217,11 +217,14 @@ For example:
   with [Go execution tracer](https://pkg.go.dev/runtime/trace) enabled,
   you may use `env GOFLAGS='-trace=trace.out' task test-integration-sqlite`.
 
-> **Note**
->
+<!-- textlint-disable one-sentence-per-line -->
+
+> [!NOTE]
 > It is not recommended to set `GOFLAGS` and other Go environment variables with `export GOFLAGS=...`
 > or `go env -w GOFLAGS=...` because they are invisible and easy to forget about, leading to confusion.
 
+<!-- textlint-enable one-sentence-per-line -->
+
 In general, we prefer integration tests over unit tests,
 tests using real databases over short tests
 and real objects over mocks.

diff --git a/README.md b/README.md
@@ -90,10 +90,14 @@ See [our documentation](https://docs.ferretdb.io/quickstart-guide/) for more det
 
 ## Building and packaging
 
-> **Note**
+<!-- textlint-disable one-sentence-per-line -->
+
+> [!NOTE]
 > We strongly advise users not to build FerretDB themselves.
 > Instead, use binaries, Docker images, or packages provided by us.
 
+<!-- textlint-enable one-sentence-per-line -->
+
 FerretDB could be built as any other Go program,
 but a few generated files and build tags could affect it.
 See [there](https://pkg.go.dev/github.com/FerretDB/FerretDB/build/version) for more details.

diff --git a/Taskfile.yml b/Taskfile.yml
@@ -203,6 +203,7 @@ tasks:
         --shard-index={{.SHARD_INDEX}}
         --shard-total={{.SHARD_TOTAL}}
         --run='{{.TEST_RUN}}'
+        --skip='{{.TEST_SKIP}}'
         --
         -count=1
         -timeout={{.TEST_TIMEOUT}}
@@ -228,6 +229,7 @@ tasks:
         --shard-index={{.SHARD_INDEX}}
         --shard-total={{.SHARD_TOTAL}}
         --run='{{.TEST_RUN}}'
+        --skip='{{.TEST_SKIP}}'
         --
         -count=1
         -timeout={{.TEST_TIMEOUT}}
@@ -253,6 +255,7 @@ tasks:
         --shard-index={{.SHARD_INDEX}}
         --shard-total={{.SHARD_TOTAL}}
         --run='{{.TEST_RUN}}'
+        --skip='{{.TEST_SKIP}}'
         --
         -count=1
         -timeout={{.TEST_TIMEOUT}}
@@ -278,6 +281,7 @@ tasks:
         --shard-index={{.SHARD_INDEX}}
         --shard-total={{.SHARD_TOTAL}}
         --run='{{.TEST_RUN}}'
+        --skip='{{.TEST_SKIP}}'
         --
         -count=1
         -timeout={{.TEST_TIMEOUT}}
@@ -303,6 +307,7 @@ tasks:
         --shard-index={{.SHARD_INDEX}}
         --shard-total={{.SHARD_TOTAL}}
         --run='{{.TEST_RUN}}'
+        --skip='{{.TEST_SKIP}}'
         --
         -count=1
         -timeout={{.TEST_TIMEOUT}}

diff --git a/build/.textlintrc b/build/.textlintrc
@@ -34,5 +34,8 @@
                 }
             ]
         }
+    },
+    "filters": {
+        "comments": true
     }
 }
diff --git a/build/deps/ferretdb-prettier.Dockerfile b/build/deps/ferretdb-prettier.Dockerfile
@@ -1 +1 @@
-FROM ghcr.io/ferretdb/ferretdb-prettier:2.8.8-1
+FROM ghcr.io/ferretdb/ferretdb-prettier:3.1.1-1
diff --git a/build/deps/ferretdb-textlint.Dockerfile b/build/deps/ferretdb-textlint.Dockerfile
@@ -1 +1 @@
-FROM ghcr.io/ferretdb/ferretdb-textlint:13.4.1-1
+FROM ghcr.io/ferretdb/ferretdb-textlint:13.4.1-2
diff --git a/cmd/envtool/envtool.go b/cmd/envtool/envtool.go
@@ -430,6 +430,7 @@ var cli struct {
 			ShardIndex uint   `help:"Shard index, starting from 1."`
 			ShardTotal uint   `help:"Total number of shards."`
 			Run        string `help:"Run only tests matching the regexp."`
+			Skip       string `help:"Skip tests matching the regexp."`
 
 			Args []string `arg:"" help:"Other arguments and flags for 'go test'." passthrough:""`
 		} `cmd:"" help:"Run tests."`
@@ -525,7 +526,7 @@ func main() {
 		err = testsRun(
 			ctx,
 			cli.Tests.Run.ShardIndex, cli.Tests.Run.ShardTotal,
-			cli.Tests.Run.Run, cli.Tests.Run.Args,
+			cli.Tests.Run.Run, cli.Tests.Run.Skip, cli.Tests.Run.Args,
 			logger,
 		)
 

diff --git a/cmd/envtool/tests.go b/cmd/envtool/tests.go
@@ -102,6 +102,7 @@ func runGoTest(ctx context.Context, args []string, total int, times bool, logger
 	}()
 
 	cmd := exec.CommandContext(ctx, "go", append([]string{"test", "-json"}, args...)...)
+
 	logger.Debugf("Running %s", strings.Join(cmd.Args, " "))
 
 	cmd.Stderr = os.Stderr
@@ -354,7 +355,7 @@ func runGoTest(ctx context.Context, args []string, total int, times bool, logger
 
 // testsRun runs tests specified by the shard index and total or by the run regex
 // using `go test` with given extra args.
-func testsRun(ctx context.Context, index, total uint, run string, args []string, logger *zap.SugaredLogger) error {
+func testsRun(ctx context.Context, index, total uint, run, skip string, args []string, logger *zap.SugaredLogger) error {
 	logger.Debugf("testsRun: index=%d, total=%d, run=%q, args=%q", index, total, run, args)
 
 	var totalTest int
@@ -386,7 +387,14 @@ func testsRun(ctx context.Context, index, total uint, run string, args []string,
 		run += ")$"
 	}
 
-	return runGoTest(ctx, append([]string{"-run=" + run}, args...), totalTest, true, logger)
+	if skip != "" {
+		totalTest = 0
+		args = append(args, "-run="+run, "-skip="+skip)
+	} else {
+		args = append(args, "-run="+run)
+	}
+
+	return runGoTest(ctx, args, totalTest, true, logger)
 }
 
 // listTestFuncs returns a sorted slice of all top-level test functions (tests, benchmarks, examples, fuzz functions)

diff --git a/go.mod b/go.mod
@@ -24,7 +24,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.21.0
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/zap v1.26.0
-	golang.org/x/crypto v0.17.0 // indirect; always use @latest
+	golang.org/x/crypto v0.17.0
 	golang.org/x/crypto/x509roots/fallback v0.0.0-20231218163308-9d2ee975ef9f
 	golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
 	golang.org/x/sys v0.15.0

diff --git a/integration/cursors/awaitdata_test.go b/integration/cursors/awaitdata_test.go
@@ -0,0 +1,123 @@
+// Copyright 2021 FerretDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cursors
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"github.com/FerretDB/FerretDB/integration"
+	"github.com/FerretDB/FerretDB/integration/setup"
+	"github.com/FerretDB/FerretDB/internal/types"
+	"github.com/FerretDB/FerretDB/internal/util/testutil"
+)
+
+func TestCursorsTailableAwaitData(tt *testing.T) {
+	tt.Parallel()
+
+	t := setup.FailsForFerretDB(tt, "https://github.com/FerretDB/FerretDB/issues/2283")
+
+	s := setup.SetupWithOpts(t, nil)
+
+	db, ctx := s.Collection.Database(), s.Ctx
+
+	opts := options.CreateCollection().SetCapped(true).SetSizeInBytes(10000000000)
+	err := db.CreateCollection(s.Ctx, testutil.CollectionName(t), opts)
+	require.NoError(t, err)
+
+	collection := db.Collection(testutil.CollectionName(t))
+
+	_, err = collection.InsertOne(ctx, bson.D{{"v", "foo"}})
+	require.NoError(t, err)
+
+	cmd := bson.D{
+		{"find", collection.Name()},
+		{"batchSize", 1},
+		{"tailable", true},
+		{"awaitData", true},
+	}
+
+	var res bson.D
+	err = collection.Database().RunCommand(ctx, cmd).Decode(&res)
+	require.NoError(t, err)
+
+	var firstBatch *types.Array
+	firstBatch, cursorID := getFirstBatch(t, res)
+
+	require.Equal(t, 1, firstBatch.Len())
+
+	getMoreCmd := bson.D{
+		{"getMore", cursorID},
+		{"collection", collection.Name()},
+		{"batchSize", 1},
+		{"maxTimeMS", (10 * time.Minute).Milliseconds()},
+	}
+
+	insertChan := make(chan error)
+
+	go func() {
+		time.Sleep(1 * time.Second)
+		_, insertErr := collection.InsertOne(ctx, bson.D{{"v", "bar"}})
+		insertChan <- insertErr
+	}()
+
+	err = collection.Database().RunCommand(ctx, getMoreCmd).Decode(&res)
+	require.NoError(t, err)
+
+	require.NoError(t, <-insertChan)
+
+	nextBatch, nextID := getNextBatch(t, res)
+	require.Equal(t, cursorID, nextID)
+	require.Equal(t, 1, nextBatch.Len())
+}
+
+func TestCursorsAwaitDataErrors(t *testing.T) {
+	t.Parallel()
+
+	s := setup.SetupWithOpts(t, nil)
+
+	db, ctx := s.Collection.Database(), s.Ctx
+
+	opts := options.CreateCollection().SetCapped(true).SetSizeInBytes(1000)
+	err := db.CreateCollection(s.Ctx, testutil.CollectionName(t), opts)
+	require.NoError(t, err)
+
+	collection := db.Collection(testutil.CollectionName(t))
+
+	_, err = collection.InsertOne(ctx, bson.D{{"v", "foo"}})
+	require.NoError(t, err)
+
+	t.Run("NonTailable", func(tt *testing.T) {
+		t := setup.FailsForFerretDB(tt, "https://github.com/FerretDB/FerretDB/issues/2283")
+
+		err = collection.Database().RunCommand(ctx, bson.D{
+			{"find", collection.Name()},
+			{"batchSize", 1},
+			{"awaitData", true},
+		}).Err()
+
+		expectedErr := mongo.CommandError{
+			Code:    9,
+			Name:    "FailedToParse",
+			Message: "Cannot set 'awaitData' without also setting 'tailable'",
+		}
+		integration.AssertEqualCommandError(t, expectedErr, err)
+	})
+}
diff --git a/integration/users/create_user_test.go b/integration/users/create_user_test.go
@@ -38,16 +38,9 @@ func TestCreateUser(t *testing.T) {
 	client := db.Client()
 	users := client.Database("admin").Collection("system.users")
 
-	// TODO https://github.com/FerretDB/FerretDB/issues/1492
-	if setup.IsMongoDB(t) {
-		assert.NoError(t, collection.Database().RunCommand(ctx, bson.D{
-			{"dropAllUsersFromDatabase", 1},
-		}).Err())
-	} else {
-		// Erase any previously saved user in the database.
-		_, err := users.DeleteMany(ctx, bson.D{{"db", db.Name()}})
-		require.NoError(t, err)
-	}
+	require.NoError(t, collection.Database().RunCommand(ctx, bson.D{
+		{"dropAllUsersFromDatabase", 1},
+	}).Err())
 
 	testCases := map[string]struct { //nolint:vet // for readability
 		payload    bson.D

diff --git a/integration/users/drop_all_users_from_database_test.go b/integration/users/drop_all_users_from_database_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 
@@ -38,7 +39,7 @@ func TestDropAllUsersFromDatabase(t *testing.T) {
 	client := collection.Database().Client()
 	users := client.Database("admin").Collection("system.users")
 
-	assert.NoError(t, collection.Database().RunCommand(ctx, bson.D{
+	require.NoError(t, collection.Database().RunCommand(ctx, bson.D{
 		{"dropAllUsersFromDatabase", 1},
 	}).Err())
 
@@ -49,7 +50,7 @@ func TestDropAllUsersFromDatabase(t *testing.T) {
 			{"roles", bson.A{}},
 			{"pwd", "password"},
 		}).Err()
-		assert.NoError(t, err)
+		require.NoError(t, err)
 	}
 
 	assertDropAllUsersFromDatabase(t, ctx, db, users, quantity)
@@ -65,7 +66,7 @@ func assertDropAllUsersFromDatabase(t *testing.T, ctx context.Context, db *mongo
 		{"dropAllUsersFromDatabase", 1},
 	}).Decode(&res)
 
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	actual := integration.ConvertDocument(t, res)
 	actual.Remove("$clusterTime")

diff --git a/integration/users/drop_user_test.go b/integration/users/drop_user_test.go
@@ -36,16 +36,9 @@ func TestDropUser(t *testing.T) {
 	client := db.Client()
 	users := client.Database("admin").Collection("system.users")
 
-	// TODO https://github.com/FerretDB/FerretDB/issues/1492
-	if setup.IsMongoDB(t) {
-		require.NoError(t, collection.Database().RunCommand(ctx, bson.D{
-			{"dropAllUsersFromDatabase", 1},
-		}).Err())
-	} else {
-		// Erase any previously saved user in the database.
-		_, err := users.DeleteMany(ctx, bson.D{{"db", db.Name()}})
-		require.NoError(t, err)
-	}
+	require.NoError(t, collection.Database().RunCommand(ctx, bson.D{
+		{"dropAllUsersFromDatabase", 1},
+	}).Err())
 
 	err := db.RunCommand(ctx, bson.D{
 		{"createUser", "a_user"},

diff --git a/internal/handler/msg_aggregate.go b/internal/handler/msg_aggregate.go
@@ -288,6 +288,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs
 		var cList *backends.ListCollectionsResult
 
 		if cList, err = db.ListCollections(ctx, nil); err != nil {
+			closer.Close()
 			return nil, err
 		}
 
@@ -312,6 +313,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs
 			}
 
 			if !cInfo.Capped() {
+				closer.Close()
 				return nil, handlererrors.NewCommandErrorMsgWithArgument(
 					handlererrors.ErrNotImplemented,
 					"$natural sort for non-capped collection is not supported.",

diff --git a/internal/handler/msg_hostinfo.go b/internal/handler/msg_hostinfo.go
@@ -72,7 +72,7 @@ func (h *Handler) MsgHostInfo(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg
 				"currentTime", now,
 				"hostname", hostname,
 				"cpuAddrSize", int32(strconv.IntSize),
-				"numCores", int32(runtime.NumCPU()),
+				"numCores", int32(runtime.GOMAXPROCS(-1)),
 				"cpuArch", runtime.GOARCH,
 			)),
 			"os", must.NotFail(types.NewDocument(
-Original file line number
+Diff line change
@@ Expand Up / @@ -34,5 +34,8 @@ @@
                     }
                 ]
             }
+        },
+        "filters": {
+            "comments": true
         }
     }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		FROM ghcr.io/ferretdb/ferretdb-prettier:2.8.8-1
		FROM ghcr.io/ferretdb/ferretdb-prettier:3.1.1-1
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		FROM ghcr.io/ferretdb/ferretdb-textlint:13.4.1-1
		FROM ghcr.io/ferretdb/ferretdb-textlint:13.4.1-2