From 913cb043ab58c8e36bb07c0b37bcd7b93f753ac8 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 21 Jun 2023 16:36:41 +0900 Subject: [PATCH 01/28] set pool size in client --- integration/query_test.go | 13 ++++++------- integration/setup/client.go | 10 +++++++--- integration/setup/listener.go | 2 +- integration/setup/setup.go | 4 +++- integration/setup/setup_compat.go | 4 ++-- integration/setup/startup.go | 4 ++-- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index af115d1968a0..938dc60ef808 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -950,7 +950,11 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() - ctx, collection := setup.Setup(t) + s := setup.SetupWithOpts(t, &setup.SetupOpts{ + PoolSize: 1, + }) + + ctx, collection := s.Ctx, s.Collection // the number of documents is set to slightly above the default batchSize of 101 docs := generateDocuments(0, 110) @@ -1179,12 +1183,7 @@ func TestQueryCommandGetMore(t *testing.T) { t.Skip(tc.skip) } - // TODO: https://github.com/FerretDB/FerretDB/issues/1807 - // Do not run tests in parallel, MongoDB throws error that session and cursor do not match. - // > Location50738 - // > Cannot run getMore on cursor 2053655655200551971, - // > which was created in session 2926eea5-9775-41a3-a563-096969f1c7d5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - , - // > in session 774d9ac6-b24a-4fd8-9874-f92ab1c9c8f5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - + t.Parallel() require.NotNil(t, tc.firstBatch, "firstBatch must not be nil") diff --git a/integration/setup/client.go b/integration/setup/client.go index a2d6a78a7132..41027c1128e6 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -78,9 +78,13 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { } // makeClient returns new client for the given working MongoDB URI. -func makeClient(ctx context.Context, uri string) (*mongo.Client, error) { +func makeClient(ctx context.Context, uri string, poolSize uint64) (*mongo.Client, error) { clientOpts := options.Client().ApplyURI(uri) + if poolSize != 0 { + clientOpts.SetMaxPoolSize(poolSize) + } + clientOpts.SetMonitor(otelmongo.NewMonitor()) client, err := mongo.Connect(ctx, clientOpts) @@ -103,7 +107,7 @@ func makeClient(ctx context.Context, uri string) (*mongo.Client, error) { // // If the connection can't be established, it panics, // as it doesn't make sense to proceed with other tests if we couldn't connect in one of them. -func setupClient(tb testing.TB, ctx context.Context, uri string) *mongo.Client { +func setupClient(tb testing.TB, ctx context.Context, uri string, poolSize uint64) *mongo.Client { tb.Helper() ctx, span := otel.Tracer("").Start(ctx, "setupClient") @@ -111,7 +115,7 @@ func setupClient(tb testing.TB, ctx context.Context, uri string) *mongo.Client { defer trace.StartRegion(ctx, "setupClient").End() - client, err := makeClient(ctx, uri) + client, err := makeClient(ctx, uri, poolSize) if err != nil { tb.Error(err) panic("setupClient: " + err.Error()) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 169fbcfcc650..ea3600317b68 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -213,7 +213,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon // those will fail the test if in-process FerretDB is not working; // for example, when backend is down uri := mongoDBURI(tb, &clientOpts) - client := setupClient(tb, ctx, uri) + client := setupClient(tb, ctx, uri, 0) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 4bc3d30dad32..c850f6fdfc50 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -86,6 +86,8 @@ type SetupOpts struct { // Benchmark data provider. If empty, collection is not created. BenchmarkProvider shareddata.BenchmarkProvider + + PoolSize uint64 } // SetupResult represents setup results. @@ -138,7 +140,7 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, ctx, logger) } else { - client = setupClient(tb, ctx, *targetURLF) + client = setupClient(tb, ctx, *targetURLF, opts.PoolSize) uri = *targetURLF } diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index a647f617cbb7..f513ce450622 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -95,7 +95,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul if *targetURLF == "" { targetClient, _ = setupListener(tb, setupCtx, logger) } else { - targetClient = setupClient(tb, setupCtx, *targetURLF) + targetClient = setupClient(tb, setupCtx, *targetURLF, 0) } // register cleanup function after setupListener registers its own to preserve full logs @@ -103,7 +103,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul targetCollections := setupCompatCollections(tb, setupCtx, targetClient, opts, *targetBackendF) - compatClient := setupClient(tb, setupCtx, *compatURLF) + compatClient := setupClient(tb, setupCtx, *compatURLF, 0) compatCollections := setupCompatCollections(tb, setupCtx, compatClient, opts, "mongodb") level.SetLevel(*logLevelF) diff --git a/integration/setup/startup.go b/integration/setup/startup.go index 124fb45714e8..019efaf50d1c 100644 --- a/integration/setup/startup.go +++ b/integration/setup/startup.go @@ -81,7 +81,7 @@ func Startup() { must.NoError(os.MkdirAll(sqliteDir, 0o777)) if u := *targetURLF; u != "" { - client, err := makeClient(ctx, u) + client, err := makeClient(ctx, u, 0) if err != nil { zap.S().Fatalf("Failed to connect to target system %s: %s", u, err) } @@ -94,7 +94,7 @@ func Startup() { } if u := *compatURLF; u != "" { - client, err := makeClient(ctx, u) + client, err := makeClient(ctx, u, 0) if err != nil { zap.S().Fatalf("Failed to connect to compat system %s: %s", u, err) } From f9168437e7d1957d4a746b2d851366fb1236736b Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 21 Jun 2023 17:19:28 +0900 Subject: [PATCH 02/28] expose ClientOption to setup --- integration/query_test.go | 2 +- integration/setup/client.go | 12 +++--------- integration/setup/listener.go | 3 ++- integration/setup/setup.go | 8 ++++++-- integration/setup/setup_compat.go | 4 ++-- integration/setup/startup.go | 5 +++-- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 938dc60ef808..e6cf1c558d5e 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -951,7 +951,7 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() s := setup.SetupWithOpts(t, &setup.SetupOpts{ - PoolSize: 1, + ClientOptions: options.Client().SetMaxPoolSize(1), }) ctx, collection := s.Ctx, s.Collection diff --git a/integration/setup/client.go b/integration/setup/client.go index 41027c1128e6..0ebd081e580d 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -78,13 +78,7 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { } // makeClient returns new client for the given working MongoDB URI. -func makeClient(ctx context.Context, uri string, poolSize uint64) (*mongo.Client, error) { - clientOpts := options.Client().ApplyURI(uri) - - if poolSize != 0 { - clientOpts.SetMaxPoolSize(poolSize) - } - +func makeClient(ctx context.Context, clientOpts *options.ClientOptions) (*mongo.Client, error) { clientOpts.SetMonitor(otelmongo.NewMonitor()) client, err := mongo.Connect(ctx, clientOpts) @@ -107,7 +101,7 @@ func makeClient(ctx context.Context, uri string, poolSize uint64) (*mongo.Client // // If the connection can't be established, it panics, // as it doesn't make sense to proceed with other tests if we couldn't connect in one of them. -func setupClient(tb testing.TB, ctx context.Context, uri string, poolSize uint64) *mongo.Client { +func setupClient(tb testing.TB, ctx context.Context, clientOpts *options.ClientOptions) *mongo.Client { tb.Helper() ctx, span := otel.Tracer("").Start(ctx, "setupClient") @@ -115,7 +109,7 @@ func setupClient(tb testing.TB, ctx context.Context, uri string, poolSize uint64 defer trace.StartRegion(ctx, "setupClient").End() - client, err := makeClient(ctx, uri, poolSize) + client, err := makeClient(ctx, clientOpts) if err != nil { tb.Error(err) panic("setupClient: " + err.Error()) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index ea3600317b68..1d53ddd419a3 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -213,7 +214,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon // those will fail the test if in-process FerretDB is not working; // for example, when backend is down uri := mongoDBURI(tb, &clientOpts) - client := setupClient(tb, ctx, uri, 0) + client := setupClient(tb, ctx, options.Client().ApplyURI(uri)) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index c850f6fdfc50..5f076f54e54f 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -87,7 +87,9 @@ type SetupOpts struct { // Benchmark data provider. If empty, collection is not created. BenchmarkProvider shareddata.BenchmarkProvider - PoolSize uint64 + // ClientOptions specifies parameters set for client. + // If MongoDB URI sets the same option, this overwrites it. + ClientOptions *options.ClientOptions } // SetupResult represents setup results. @@ -140,7 +142,9 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, ctx, logger) } else { - client = setupClient(tb, ctx, *targetURLF, opts.PoolSize) + uriClientOpts := options.Client().ApplyURI(*targetURLF) + clientOpts := options.MergeClientOptions(uriClientOpts, opts.ClientOptions) + client = setupClient(tb, ctx, clientOpts) uri = *targetURLF } diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index f513ce450622..88628e53c2ac 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -95,7 +95,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul if *targetURLF == "" { targetClient, _ = setupListener(tb, setupCtx, logger) } else { - targetClient = setupClient(tb, setupCtx, *targetURLF, 0) + targetClient = setupClient(tb, setupCtx, options.Client().ApplyURI(*targetURLF)) } // register cleanup function after setupListener registers its own to preserve full logs @@ -103,7 +103,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul targetCollections := setupCompatCollections(tb, setupCtx, targetClient, opts, *targetBackendF) - compatClient := setupClient(tb, setupCtx, *compatURLF, 0) + compatClient := setupClient(tb, setupCtx, options.Client().ApplyURI(*compatURLF)) compatCollections := setupCompatCollections(tb, setupCtx, compatClient, opts, "mongodb") level.SetLevel(*logLevelF) diff --git a/integration/setup/startup.go b/integration/setup/startup.go index 019efaf50d1c..f3ba2b072b0c 100644 --- a/integration/setup/startup.go +++ b/integration/setup/startup.go @@ -23,6 +23,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "go.mongodb.org/mongo-driver/mongo/options" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/jaeger" "go.opentelemetry.io/otel/sdk/resource" @@ -81,7 +82,7 @@ func Startup() { must.NoError(os.MkdirAll(sqliteDir, 0o777)) if u := *targetURLF; u != "" { - client, err := makeClient(ctx, u, 0) + client, err := makeClient(ctx, options.Client().ApplyURI(u)) if err != nil { zap.S().Fatalf("Failed to connect to target system %s: %s", u, err) } @@ -94,7 +95,7 @@ func Startup() { } if u := *compatURLF; u != "" { - client, err := makeClient(ctx, u, 0) + client, err := makeClient(ctx, options.Client().ApplyURI(u)) if err != nil { zap.S().Fatalf("Failed to connect to compat system %s: %s", u, err) } From 3e00842376d12701aa0c4f1813a8c14ee20f3393 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 23 Jun 2023 11:26:19 +0900 Subject: [PATCH 03/28] rename and comment update --- integration/setup/client.go | 2 +- integration/setup/setup.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index 0ebd081e580d..ada286351774 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -77,7 +77,7 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { return u.String() } -// makeClient returns new client for the given working MongoDB URI. +// makeClient returns new client for the given MongoDB client options. func makeClient(ctx context.Context, clientOpts *options.ClientOptions) (*mongo.Client, error) { clientOpts.SetMonitor(otelmongo.NewMonitor()) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 5f076f54e54f..afad18538f77 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -87,8 +87,7 @@ type SetupOpts struct { // Benchmark data provider. If empty, collection is not created. BenchmarkProvider shareddata.BenchmarkProvider - // ClientOptions specifies parameters set for client. - // If MongoDB URI sets the same option, this overwrites it. + // ClientOptions overwrites the options set in MongoDB URI when the same option is set. ClientOptions *options.ClientOptions } @@ -142,8 +141,8 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, ctx, logger) } else { - uriClientOpts := options.Client().ApplyURI(*targetURLF) - clientOpts := options.MergeClientOptions(uriClientOpts, opts.ClientOptions) + clientOpts := options.Client().ApplyURI(*targetURLF) + clientOpts = options.MergeClientOptions(clientOpts, opts.ClientOptions) client = setupClient(tb, ctx, clientOpts) uri = *targetURLF } From 04c75a614cb1a942f6f8188af737efd90e55ba0c Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 23 Jun 2023 17:22:58 +0900 Subject: [PATCH 04/28] use url.Values --- integration/query_test.go | 8 +++++++- integration/setup/client.go | 25 +++++++++++++++++++++---- integration/setup/listener.go | 3 +-- integration/setup/setup.go | 9 ++++----- integration/setup/setup_compat.go | 4 ++-- integration/setup/startup.go | 5 ++--- 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index e6cf1c558d5e..99855a4fdca6 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -16,6 +16,7 @@ package integration import ( "math" + "net/url" "testing" "time" @@ -950,8 +951,13 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() + + q := url.Values{} + q.Set("maxPoolSize", "1") + q.Set("minPoolSize", "1") + s := setup.SetupWithOpts(t, &setup.SetupOpts{ - ClientOptions: options.Client().SetMaxPoolSize(1), + ExtraOptions: q, }) ctx, collection := s.Ctx, s.Collection diff --git a/integration/setup/client.go b/integration/setup/client.go index ada286351774..6b90f71b1ef3 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -77,8 +77,25 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { return u.String() } -// makeClient returns new client for the given MongoDB client options. -func makeClient(ctx context.Context, clientOpts *options.ClientOptions) (*mongo.Client, error) { +// makeClient returns new client for the given MongoDB URI and extra options. +func makeClient(ctx context.Context, uri string, extraOpts url.Values) (*mongo.Client, error) { + u, err := url.Parse(uri) + if err != nil { + return nil, err + } + + q := u.Query() + + for k, v := range extraOpts { + if len(v) == 0 { + continue + } + + // when multiple values are assigned for the same key, use the first one only + q.Set(k, v[0]) + } + + clientOpts := options.Client().ApplyURI(u.String()) clientOpts.SetMonitor(otelmongo.NewMonitor()) client, err := mongo.Connect(ctx, clientOpts) @@ -101,7 +118,7 @@ func makeClient(ctx context.Context, clientOpts *options.ClientOptions) (*mongo. // // If the connection can't be established, it panics, // as it doesn't make sense to proceed with other tests if we couldn't connect in one of them. -func setupClient(tb testing.TB, ctx context.Context, clientOpts *options.ClientOptions) *mongo.Client { +func setupClient(tb testing.TB, ctx context.Context, uri string, extraOpts url.Values) *mongo.Client { tb.Helper() ctx, span := otel.Tracer("").Start(ctx, "setupClient") @@ -109,7 +126,7 @@ func setupClient(tb testing.TB, ctx context.Context, clientOpts *options.ClientO defer trace.StartRegion(ctx, "setupClient").End() - client, err := makeClient(ctx, clientOpts) + client, err := makeClient(ctx, uri, extraOpts) if err != nil { tb.Error(err) panic("setupClient: " + err.Error()) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 1d53ddd419a3..06845239df26 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -214,7 +213,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon // those will fail the test if in-process FerretDB is not working; // for example, when backend is down uri := mongoDBURI(tb, &clientOpts) - client := setupClient(tb, ctx, options.Client().ApplyURI(uri)) + client := setupClient(tb, ctx, uri, nil) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index afad18538f77..1858fd245518 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -19,6 +19,7 @@ import ( "context" "flag" "fmt" + "net/url" "path/filepath" "runtime/trace" "strings" @@ -87,8 +88,8 @@ type SetupOpts struct { // Benchmark data provider. If empty, collection is not created. BenchmarkProvider shareddata.BenchmarkProvider - // ClientOptions overwrites the options set in MongoDB URI when the same option is set. - ClientOptions *options.ClientOptions + // ExtraOptions overwrites the options set in MongoDB URI when the same option is set. + ExtraOptions url.Values } // SetupResult represents setup results. @@ -141,9 +142,7 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, ctx, logger) } else { - clientOpts := options.Client().ApplyURI(*targetURLF) - clientOpts = options.MergeClientOptions(clientOpts, opts.ClientOptions) - client = setupClient(tb, ctx, clientOpts) + client = setupClient(tb, ctx, *targetURLF, opts.ExtraOptions) uri = *targetURLF } diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index 88628e53c2ac..e556b56aada4 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -95,7 +95,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul if *targetURLF == "" { targetClient, _ = setupListener(tb, setupCtx, logger) } else { - targetClient = setupClient(tb, setupCtx, options.Client().ApplyURI(*targetURLF)) + targetClient = setupClient(tb, setupCtx, *targetURLF, nil) } // register cleanup function after setupListener registers its own to preserve full logs @@ -103,7 +103,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul targetCollections := setupCompatCollections(tb, setupCtx, targetClient, opts, *targetBackendF) - compatClient := setupClient(tb, setupCtx, options.Client().ApplyURI(*compatURLF)) + compatClient := setupClient(tb, setupCtx, *compatURLF, nil) compatCollections := setupCompatCollections(tb, setupCtx, compatClient, opts, "mongodb") level.SetLevel(*logLevelF) diff --git a/integration/setup/startup.go b/integration/setup/startup.go index f3ba2b072b0c..a0308f4386ac 100644 --- a/integration/setup/startup.go +++ b/integration/setup/startup.go @@ -23,7 +23,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "go.mongodb.org/mongo-driver/mongo/options" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/jaeger" "go.opentelemetry.io/otel/sdk/resource" @@ -82,7 +81,7 @@ func Startup() { must.NoError(os.MkdirAll(sqliteDir, 0o777)) if u := *targetURLF; u != "" { - client, err := makeClient(ctx, options.Client().ApplyURI(u)) + client, err := makeClient(ctx, u, nil) if err != nil { zap.S().Fatalf("Failed to connect to target system %s: %s", u, err) } @@ -95,7 +94,7 @@ func Startup() { } if u := *compatURLF; u != "" { - client, err := makeClient(ctx, options.Client().ApplyURI(u)) + client, err := makeClient(ctx, u, nil) if err != nil { zap.S().Fatalf("Failed to connect to compat system %s: %s", u, err) } From 40a511f59ad3dfdf4a841d1cb76c38b8ae7e993d Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 10:41:29 +0900 Subject: [PATCH 05/28] tidy up --- integration/query_test.go | 2 ++ integration/setup/client.go | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 99855a4fdca6..2eebea784d40 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -953,6 +953,8 @@ func TestQueryCommandGetMore(t *testing.T) { t.Parallel() q := url.Values{} + + // Set 1 to ensure only one pool exists duration of the test q.Set("maxPoolSize", "1") q.Set("minPoolSize", "1") diff --git a/integration/setup/client.go b/integration/setup/client.go index 6b90f71b1ef3..660438cd9ff7 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -86,15 +86,13 @@ func makeClient(ctx context.Context, uri string, extraOpts url.Values) (*mongo.C q := u.Query() - for k, v := range extraOpts { - if len(v) == 0 { - continue + for k, vs := range extraOpts { + for _, v := range vs { + q.Set(k, v) } - - // when multiple values are assigned for the same key, use the first one only - q.Set(k, v[0]) } + u.RawQuery = q.Encode() clientOpts := options.Client().ApplyURI(u.String()) clientOpts.SetMonitor(otelmongo.NewMonitor()) From 2f1b735828f5b9b68842a8d4af1ce9f182c89252 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 11:37:40 +0900 Subject: [PATCH 06/28] add same client and different client connection tests --- integration/commands_diagnostic_test.go | 120 ++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 672351dbb785..747c1408a014 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -16,6 +16,9 @@ package integration import ( "net" + "net/url" + "runtime" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -406,3 +409,120 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { assert.NotEqual(t, ports[0], ports[1]) } } + +func TestCommandWhatsMyURIConnection(t *testing.T) { + t.Parallel() + + // Set 1 to ensure only one pool exists duration of the test + q1 := url.Values{} + q1.Set("maxPoolSize", "1") + q1.Set("minPoolSize", "1") + + s := setup.SetupWithOpts(t, &setup.SetupOpts{ + ExtraOptions: q1, + }) + + collection1 := s.Collection + databaseName := s.Collection.Database().Name() + collectionName := s.Collection.Name() + + t.Run("SameClient", func(t *testing.T) { + num := runtime.GOMAXPROCS(-1) * 10 + ready := make(chan struct{}, num) + start := make(chan struct{}) + ports := make(chan string, num) + + var wg sync.WaitGroup + for i := 0; i < num; i++ { + wg.Add(1) + + go func(i int) { + defer wg.Done() + + ready <- struct{}{} + + <-start + + var res bson.D + err := collection1.Database().RunCommand(s.Ctx, bson.D{{"whatsmyuri", int32(1)}}).Decode(&res) + require.NoError(t, err) + + doc := ConvertDocument(t, res) + v, _ := doc.Get("ok") + resOk, ok := v.(float64) + require.True(t, ok) + assert.Equal(t, float64(1), resOk) + + v, _ = doc.Get("you") + you, ok := v.(string) + require.True(t, ok) + + _, port, err := net.SplitHostPort(you) + require.NoError(t, err) + assert.NotEmpty(t, port) + ports <- port + }(i) + } + + for i := 0; i < num; i++ { + <-ready + } + + close(start) + + wg.Wait() + + close(ports) + + previousPort := <-ports + for port := range ports { + assert.Equal(t, previousPort, port) + previousPort = port + } + }) + + t.Run("DifferentClient", func(t *testing.T) { + u, err := url.Parse(s.MongoDBURI) + require.NoError(t, err) + + q2 := u.Query() + q2.Set("maxPoolSize", "1") + q2.Set("minPoolSize", "1") + u.RawQuery = q2.Encode() + + client2, err := mongo.Connect(s.Ctx, options.Client().ApplyURI(u.String())) + require.NoError(t, err) + + defer client2.Disconnect(s.Ctx) + + collection2 := client2.Database(databaseName).Collection(collectionName) + + var ports []string + + for _, collection := range []*mongo.Collection{collection1, collection2} { + var res bson.D + err := collection.Database().RunCommand(s.Ctx, bson.D{{"whatsmyuri", int32(1)}}).Decode(&res) + require.NoError(t, err) + + doc := ConvertDocument(t, res) + v, _ := doc.Get("ok") + resOk, ok := v.(float64) + require.True(t, ok) + assert.Equal(t, float64(1), resOk) + + v, _ = doc.Get("you") + you, ok := v.(string) + require.True(t, ok) + + _, port, err := net.SplitHostPort(you) + require.NoError(t, err) + assert.NotEmpty(t, port) + + ports = append(ports, port) + } + + // compare ports from two different clients are not equal. + require.Equal(t, 2, len(ports)) + assert.NotEqual(t, ports[0], ports[1]) + }) +} From 5d0a1476e738e1695b1102d1e570f2df47f2b51e Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 11:55:11 +0900 Subject: [PATCH 07/28] update error message for same client tests --- integration/commands_diagnostic_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 747c1408a014..aa1ea540a6e9 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -474,10 +474,13 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { close(ports) - previousPort := <-ports + var allPorts []string for port := range ports { - assert.Equal(t, previousPort, port) - previousPort = port + allPorts = append(allPorts, port) + } + + for _, port := range allPorts { + require.Equal(t, allPorts[0], port, "expected same client uses the same port: %s", allPorts) } }) From 9de04ca18256d2244fa9a3a83d82d892df28bce3 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 12:15:36 +0900 Subject: [PATCH 08/28] skip test for feature not supported by FerretDB --- integration/commands_diagnostic_test.go | 9 ++++++++- integration/query_test.go | 3 ++- integration/setup/test_helpers.go | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index aa1ea540a6e9..412161e730e0 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -410,10 +410,15 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { } } +// TestCommandWhatsMyURIConnection tests that a client uses the same connection for +// all commands run by the client, and different clients use different connection. +// The port number is used to validate if the same connection is used or not, +// since unique port number is assigned to each client in the test setup. func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() - // Set 1 to ensure only one pool exists duration of the test + // set 1 to ensure only one pool exists duration of the test, + // which forces a client to use a single pool q1 := url.Values{} q1.Set("maxPoolSize", "1") q1.Set("minPoolSize", "1") @@ -427,6 +432,8 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { collectionName := s.Collection.Name() t.Run("SameClient", func(t *testing.T) { + setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/2906") + num := runtime.GOMAXPROCS(-1) * 10 ready := make(chan struct{}, num) start := make(chan struct{}) diff --git a/integration/query_test.go b/integration/query_test.go index 2eebea784d40..983ba5190c14 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -954,7 +954,8 @@ func TestQueryCommandGetMore(t *testing.T) { q := url.Values{} - // Set 1 to ensure only one pool exists duration of the test + // set 1 to ensure only one pool exists duration of the test, + // which forces a client to use a single pool q.Set("maxPoolSize", "1") q.Set("minPoolSize", "1") diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index f16c5e8ab2e1..1b152246543f 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -33,6 +33,19 @@ func SkipForMongoDB(tb testing.TB, reason string) { } } +// SkipExceptMongoDB skips the current test for backends except MongoDB. +// +// This function should not be used lightly. +func SkipExceptMongoDB(tb testing.TB, reason string) { + tb.Helper() + + if *targetBackendF != "mongodb" { + require.NotEmpty(tb, reason, "reason must not be empty") + + tb.Skipf("Skipping for MongoDB: %s.", reason) + } +} + // IsTigris returns true if tests are running against FerretDB with `ferretdb-tigris` backend. // // This function should not be used lightly. From 734d5bfe05ca39556bfe93cf3fbda9d3d66d0820 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 12:25:24 +0900 Subject: [PATCH 09/28] comment update --- integration/commands_diagnostic_test.go | 5 ++--- integration/query_test.go | 2 +- integration/setup/client.go | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 412161e730e0..657fdb1434d4 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -413,12 +413,12 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { // TestCommandWhatsMyURIConnection tests that a client uses the same connection for // all commands run by the client, and different clients use different connection. // The port number is used to validate if the same connection is used or not, -// since unique port number is assigned to each client in the test setup. +// since unique port number is assigned to each client connection in the test setup. func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() // set 1 to ensure only one pool exists duration of the test, - // which forces a client to use a single pool + // which forces a client to use a single connection pool q1 := url.Values{} q1.Set("maxPoolSize", "1") q1.Set("minPoolSize", "1") @@ -531,7 +531,6 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { ports = append(ports, port) } - // compare ports from two different clients are not equal. require.Equal(t, 2, len(ports)) assert.NotEqual(t, ports[0], ports[1]) }) diff --git a/integration/query_test.go b/integration/query_test.go index 983ba5190c14..1a5bc2a1cdcf 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -955,7 +955,7 @@ func TestQueryCommandGetMore(t *testing.T) { q := url.Values{} // set 1 to ensure only one pool exists duration of the test, - // which forces a client to use a single pool + // which forces a client to use a single connection pool q.Set("maxPoolSize", "1") q.Set("minPoolSize", "1") diff --git a/integration/setup/client.go b/integration/setup/client.go index 660438cd9ff7..0f09d478384c 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -78,6 +78,7 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { } // makeClient returns new client for the given MongoDB URI and extra options. +// The extraOpts that comes later overwrites value for the query key. func makeClient(ctx context.Context, uri string, extraOpts url.Values) (*mongo.Client, error) { u, err := url.Parse(uri) if err != nil { From 962ed963eabd64721b16b75809001e5c3f6386e4 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 26 Jun 2023 13:57:46 +0900 Subject: [PATCH 10/28] run subtests in parallel --- integration/commands_diagnostic_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 657fdb1434d4..751bf75bc8a1 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -434,6 +434,8 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { t.Run("SameClient", func(t *testing.T) { setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/2906") + t.Parallel() + num := runtime.GOMAXPROCS(-1) * 10 ready := make(chan struct{}, num) start := make(chan struct{}) @@ -492,6 +494,8 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { }) t.Run("DifferentClient", func(t *testing.T) { + t.Parallel() + u, err := url.Parse(s.MongoDBURI) require.NoError(t, err) From b0e876149305945248ae9116a8c45d33df87d6d7 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 13:38:35 +0900 Subject: [PATCH 11/28] test setup handles merging url with extra url.Values option --- integration/commands_diagnostic_test.go | 11 +++++------ integration/setup/client.go | 25 +++++-------------------- integration/setup/listener.go | 2 +- integration/setup/setup.go | 13 ++++++++++++- integration/setup/setup_compat.go | 4 ++-- integration/setup/startup.go | 4 ++-- 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 85a0ad18d676..af8b8e66f418 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -415,14 +415,13 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() - // set 1 to ensure only one pool exists duration of the test, + // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, // which forces a client to use a single connection pool - q1 := url.Values{} - q1.Set("maxPoolSize", "1") - q1.Set("minPoolSize", "1") - s := setup.SetupWithOpts(t, &setup.SetupOpts{ - ExtraOptions: q1, + ExtraOptions: url.Values{ + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + }, }) collection1 := s.Collection diff --git a/integration/setup/client.go b/integration/setup/client.go index beba11911483..f0646c8b27fe 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -77,24 +77,9 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { return u.String() } -// makeClient returns new client for the given MongoDB URI and extra options. -// The extraOpts that comes later overwrites value for the query key. -func makeClient(ctx context.Context, uri string, extraOpts url.Values) (*mongo.Client, error) { - u, err := url.Parse(uri) - if err != nil { - return nil, err - } - - q := u.Query() - - for k, vs := range extraOpts { - for _, v := range vs { - q.Set(k, v) - } - } - - u.RawQuery = q.Encode() - clientOpts := options.Client().ApplyURI(u.String()) +// makeClient returns new client for the given working MongoDB URI. +func makeClient(ctx context.Context, uri string) (*mongo.Client, error) { + clientOpts := options.Client().ApplyURI(uri) clientOpts.SetMonitor(otelmongo.NewMonitor()) client, err := mongo.Connect(ctx, clientOpts) @@ -117,7 +102,7 @@ func makeClient(ctx context.Context, uri string, extraOpts url.Values) (*mongo.C // // If the connection can't be established, it panics, // as it doesn't make sense to proceed with other tests if we couldn't connect in one of them. -func setupClient(tb testing.TB, ctx context.Context, uri string, extraOpts url.Values) *mongo.Client { +func setupClient(tb testing.TB, ctx context.Context, uri string) *mongo.Client { tb.Helper() ctx, span := otel.Tracer("").Start(ctx, "setupClient") @@ -125,7 +110,7 @@ func setupClient(tb testing.TB, ctx context.Context, uri string, extraOpts url.V defer observability.FuncCall(ctx)() - client, err := makeClient(ctx, uri, extraOpts) + client, err := makeClient(ctx, uri) if err != nil { tb.Error(err) panic("setupClient: " + err.Error()) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index a4562deb37bd..6600113202bc 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -212,7 +212,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon // those will fail the test if in-process FerretDB is not working; // for example, when backend is down uri := mongoDBURI(tb, &clientOpts) - client := setupClient(tb, ctx, uri, nil) + client := setupClient(tb, ctx, uri) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index dd22b5ae6359..197174838408 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -143,7 +143,18 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, setupCtx, logger) } else { - client = setupClient(tb, setupCtx, *targetURLF, opts.ExtraOptions) + u, err := url.Parse(*targetURLF) + require.NoError(tb, err) + + q := u.Query() + for k, vs := range opts.ExtraOptions { + for _, v := range vs { + q.Set(k, v) + } + } + + u.RawQuery = q.Encode() + client = setupClient(tb, setupCtx, u.String()) uri = *targetURLF } diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index 2ea022afcac6..f6edfaccc867 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -96,7 +96,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul if *targetURLF == "" { targetClient, _ = setupListener(tb, setupCtx, logger) } else { - targetClient = setupClient(tb, setupCtx, *targetURLF, nil) + targetClient = setupClient(tb, setupCtx, *targetURLF) } // register cleanup function after setupListener registers its own to preserve full logs @@ -104,7 +104,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul targetCollections := setupCompatCollections(tb, setupCtx, targetClient, opts, *targetBackendF) - compatClient := setupClient(tb, setupCtx, *compatURLF, nil) + compatClient := setupClient(tb, setupCtx, *compatURLF) compatCollections := setupCompatCollections(tb, setupCtx, compatClient, opts, "mongodb") level.SetLevel(*logLevelF) diff --git a/integration/setup/startup.go b/integration/setup/startup.go index a0308f4386ac..124fb45714e8 100644 --- a/integration/setup/startup.go +++ b/integration/setup/startup.go @@ -81,7 +81,7 @@ func Startup() { must.NoError(os.MkdirAll(sqliteDir, 0o777)) if u := *targetURLF; u != "" { - client, err := makeClient(ctx, u, nil) + client, err := makeClient(ctx, u) if err != nil { zap.S().Fatalf("Failed to connect to target system %s: %s", u, err) } @@ -94,7 +94,7 @@ func Startup() { } if u := *compatURLF; u != "" { - client, err := makeClient(ctx, u, nil) + client, err := makeClient(ctx, u) if err != nil { zap.S().Fatalf("Failed to connect to compat system %s: %s", u, err) } From 733b14494898727e2001e60d1f30792eff3c56ab Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 13:56:12 +0900 Subject: [PATCH 12/28] use stresstest.Stress --- integration/commands_diagnostic_test.go | 64 +++++++++---------------- internal/util/teststress/stress.go | 11 +++-- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index af8b8e66f418..52a08057cafa 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -17,8 +17,6 @@ package integration import ( "net" "net/url" - "runtime" - "sync" "testing" "github.com/stretchr/testify/assert" @@ -31,6 +29,7 @@ import ( "github.com/FerretDB/FerretDB/integration/shareddata" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/must" + "github.com/FerretDB/FerretDB/internal/util/teststress" "github.com/FerretDB/FerretDB/internal/util/testutil" ) @@ -428,55 +427,36 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { databaseName := s.Collection.Database().Name() collectionName := s.Collection.Name() - t.Run("SameClient", func(t *testing.T) { + t.Run("SameClientStress", func(t *testing.T) { setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/2906") t.Parallel() - num := runtime.GOMAXPROCS(-1) * 10 - ready := make(chan struct{}, num) - start := make(chan struct{}) - ports := make(chan string, num) + ports := make(chan string, teststress.NumGoroutines) - var wg sync.WaitGroup - for i := 0; i < num; i++ { - wg.Add(1) + teststress.Stress(t, func(ready chan<- struct{}, start <-chan struct{}) { + ready <- struct{}{} + <-start - go func(i int) { - defer wg.Done() - - ready <- struct{}{} - - <-start - - var res bson.D - err := collection1.Database().RunCommand(s.Ctx, bson.D{{"whatsmyuri", int32(1)}}).Decode(&res) - require.NoError(t, err) - - doc := ConvertDocument(t, res) - v, _ := doc.Get("ok") - resOk, ok := v.(float64) - require.True(t, ok) - assert.Equal(t, float64(1), resOk) - - v, _ = doc.Get("you") - you, ok := v.(string) - require.True(t, ok) - - _, port, err := net.SplitHostPort(you) - require.NoError(t, err) - assert.NotEmpty(t, port) - ports <- port - }(i) - } + var res bson.D + err := collection1.Database().RunCommand(s.Ctx, bson.D{{"whatsmyuri", int32(1)}}).Decode(&res) + require.NoError(t, err) - for i := 0; i < num; i++ { - <-ready - } + doc := ConvertDocument(t, res) + v, _ := doc.Get("ok") + resOk, ok := v.(float64) + require.True(t, ok) + assert.Equal(t, float64(1), resOk) - close(start) + v, _ = doc.Get("you") + you, ok := v.(string) + require.True(t, ok) - wg.Wait() + _, port, err := net.SplitHostPort(you) + require.NoError(t, err) + assert.NotEmpty(t, port) + ports <- port + }) close(ports) diff --git a/internal/util/teststress/stress.go b/internal/util/teststress/stress.go index 972c95ef0daa..794d730224e0 100644 --- a/internal/util/teststress/stress.go +++ b/internal/util/teststress/stress.go @@ -23,6 +23,9 @@ import ( "testing" ) +// NumGoroutines is the total count of goroutines created in Stress function. +var NumGoroutines = runtime.GOMAXPROCS(-1) * 10 + // Stress runs function f in multiple goroutines. // // Function f should do a needed setup, send a message to ready channel when it is ready to start, @@ -30,15 +33,13 @@ import ( func Stress(tb testing.TB, f func(ready chan<- struct{}, start <-chan struct{})) { tb.Helper() - n := runtime.GOMAXPROCS(-1) * 10 - // do a bit more work to reduce a chance that one goroutine would finish // before the other one is still being created var wg sync.WaitGroup - readyCh := make(chan struct{}, n) + readyCh := make(chan struct{}, NumGoroutines) startCh := make(chan struct{}) - for i := 0; i < n; i++ { + for i := 0; i < NumGoroutines; i++ { wg.Add(1) go func() { @@ -48,7 +49,7 @@ func Stress(tb testing.TB, f func(ready chan<- struct{}, start <-chan struct{})) }() } - for i := 0; i < n; i++ { + for i := 0; i < NumGoroutines; i++ { <-readyCh } From e6a30e5bd4105f7a0248c1f525c8db5a8e729c2d Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 15:29:53 +0900 Subject: [PATCH 13/28] add batchSize test for find and getMore --- integration/query_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/integration/query_test.go b/integration/query_test.go index 5c7c4c33b7c9..79d05453fb21 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -1192,6 +1192,34 @@ func TestQueryCommandGetMore(t *testing.T) { Message: "BSON field 'getMore.collection' is missing but a required field", }, }, + "UnsetAllBatchSize": { + findBatchSize: nil, + getMoreBatchSize: nil, + collection: collection.Name(), + firstBatch: docs[:101], + nextBatch: docs[101:], + }, + "UnsetFindBatchSize": { + findBatchSize: nil, + getMoreBatchSize: 5, + collection: collection.Name(), + firstBatch: docs[:101], + nextBatch: docs[101:106], + }, + "UnsetGetMoreBatchSize": { + findBatchSize: 5, + getMoreBatchSize: nil, + collection: collection.Name(), + firstBatch: docs[:5], + nextBatch: docs[5:], + }, + "BatchSize": { + findBatchSize: 3, + getMoreBatchSize: 5, + collection: collection.Name(), + firstBatch: docs[:3], + nextBatch: docs[3:8], + }, } { name, tc := name, tc t.Run(name, func(t *testing.T) { From c6811fe723a4cfefd7e650aad245c14363d34e27 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 16:14:32 +0900 Subject: [PATCH 14/28] add getMore test for using different client --- integration/commands_diagnostic_test.go | 2 - integration/query_test.go | 126 ++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 52a08057cafa..73e8a16d2eba 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -428,8 +428,6 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { collectionName := s.Collection.Name() t.Run("SameClientStress", func(t *testing.T) { - setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/2906") - t.Parallel() ports := make(chan string, teststress.NumGoroutines) diff --git a/integration/query_test.go b/integration/query_test.go index 79d05453fb21..11937d46f7d2 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -30,6 +30,7 @@ import ( "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" + "github.com/FerretDB/FerretDB/internal/util/teststress" ) func TestQueryBadFindType(t *testing.T) { @@ -958,15 +959,13 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() - q := url.Values{} - - // set 1 to ensure only one pool exists duration of the test, + // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, // which forces a client to use a single connection pool - q.Set("maxPoolSize", "1") - q.Set("minPoolSize", "1") - s := setup.SetupWithOpts(t, &setup.SetupOpts{ - ExtraOptions: q, + ExtraOptions: url.Values{ + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + }, }) ctx, collection := s.Ctx, s.Collection @@ -1316,3 +1315,116 @@ func TestQueryCommandGetMore(t *testing.T) { }) } } + +// TestQueryCommandGetMoreConnection tests that a client uses the same connection for +// getMore command run by the client, and different clients use different connection. +func TestQueryCommandGetMoreConnection(t *testing.T) { + t.Parallel() + + // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, + // which forces a client to use a single connection pool + s := setup.SetupWithOpts(t, &setup.SetupOpts{ + ExtraOptions: url.Values{ + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + }, + }) + + collection1 := s.Collection + databaseName := s.Collection.Database().Name() + collectionName := s.Collection.Name() + + docs := generateDocuments(0, 110) + _, err := collection1.InsertMany(s.Ctx, docs) + require.NoError(t, err) + + t.Run("SameClientStress", func(t *testing.T) { + t.Parallel() + + teststress.Stress(t, func(ready chan<- struct{}, start <-chan struct{}) { + ready <- struct{}{} + <-start + + var res bson.D + err := collection1.Database().RunCommand(s.Ctx, + bson.D{ + {"find", collection1.Name()}, + {"batchSize", 2}, + }).Decode(&res) + require.NoError(t, err) + + v, ok := res.Map()["cursor"] + require.True(t, ok) + + cursor, ok := v.(bson.D) + require.True(t, ok) + + cursorID := cursor.Map()["id"] + assert.NotNil(t, cursorID) + + err = collection1.Database().RunCommand(s.Ctx, bson.D{ + {"getMore", cursorID}, + {"collection", collection1.Name()}, + }).Decode(&res) + require.NoError(t, err) + }) + }) + + t.Run("DifferentClient", func(t *testing.T) { + // error returned from using different clients are session related error, + // currently FerretDB does not return an error + setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/153") + + t.Parallel() + + u, err := url.Parse(s.MongoDBURI) + require.NoError(t, err) + + q2 := u.Query() + q2.Set("maxPoolSize", "1") + q2.Set("minPoolSize", "1") + u.RawQuery = q2.Encode() + + client2, err := mongo.Connect(s.Ctx, options.Client().ApplyURI(u.String())) + require.NoError(t, err) + + defer client2.Disconnect(s.Ctx) + + collection2 := client2.Database(databaseName).Collection(collectionName) + + var res bson.D + err = collection1.Database().RunCommand(s.Ctx, + bson.D{ + {"find", collection1.Name()}, + {"batchSize", 2}, + }).Decode(&res) + require.NoError(t, err) + + v, ok := res.Map()["cursor"] + require.True(t, ok) + + cursor, ok := v.(bson.D) + require.True(t, ok) + + cursorID := cursor.Map()["id"] + assert.NotNil(t, cursorID) + + err = collection2.Database().RunCommand(s.Ctx, bson.D{ + {"getMore", cursorID}, + {"collection", collection2.Name()}, + }).Decode(&res) + + // use AssertEqualCommandError because message cannot be compared as it contains specific session ID + AssertMatchesCommandError( + t, + mongo.CommandError{ + Code: 50738, + Name: "Location50738", + Message: "Cannot run getMore on cursor 5720627396082469624, which was created in session " + + "95326129-ff9c-48a4-9060-464b4ea3ee06 - 47DEQpj8HBSa+/TImW+5JC\neuQeRkm5NMpJWZG3hSuFU= - - , " + + "in session 9e8902e9-338c-4156-9fd8-50e5d62ac992 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - ", + }, + err, + ) + }) +} From 1b450c7345f33b05ac72416aeb5b413d56d84be7 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 16:23:41 +0900 Subject: [PATCH 15/28] update test --- integration/query_test.go | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 11937d46f7d2..111a5a4a55b4 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -30,7 +30,6 @@ import ( "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" - "github.com/FerretDB/FerretDB/internal/util/teststress" ) func TestQueryBadFindType(t *testing.T) { @@ -1316,8 +1315,6 @@ func TestQueryCommandGetMore(t *testing.T) { } } -// TestQueryCommandGetMoreConnection tests that a client uses the same connection for -// getMore command run by the client, and different clients use different connection. func TestQueryCommandGetMoreConnection(t *testing.T) { t.Parallel() @@ -1334,42 +1331,10 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { databaseName := s.Collection.Database().Name() collectionName := s.Collection.Name() - docs := generateDocuments(0, 110) + docs := generateDocuments(0, 5) _, err := collection1.InsertMany(s.Ctx, docs) require.NoError(t, err) - t.Run("SameClientStress", func(t *testing.T) { - t.Parallel() - - teststress.Stress(t, func(ready chan<- struct{}, start <-chan struct{}) { - ready <- struct{}{} - <-start - - var res bson.D - err := collection1.Database().RunCommand(s.Ctx, - bson.D{ - {"find", collection1.Name()}, - {"batchSize", 2}, - }).Decode(&res) - require.NoError(t, err) - - v, ok := res.Map()["cursor"] - require.True(t, ok) - - cursor, ok := v.(bson.D) - require.True(t, ok) - - cursorID := cursor.Map()["id"] - assert.NotNil(t, cursorID) - - err = collection1.Database().RunCommand(s.Ctx, bson.D{ - {"getMore", cursorID}, - {"collection", collection1.Name()}, - }).Decode(&res) - require.NoError(t, err) - }) - }) - t.Run("DifferentClient", func(t *testing.T) { // error returned from using different clients are session related error, // currently FerretDB does not return an error From 405e8c960339279b607bb1685d5b4232e4a2b984 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 17:03:28 +0900 Subject: [PATCH 16/28] update commnet --- integration/query_test.go | 55 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 111a5a4a55b4..c621460de888 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -1276,6 +1276,18 @@ func TestQueryCommandGetMore(t *testing.T) { getMoreRest..., ) + // Even `find` and `getMore` use the same connection by setting `minPoolSize` and `maxPoolSize` to 1, + // on rare occasion it may return session error due to different session calling `find` and `getMore`. + // If that happens too frequently, removing t.Parallel() in subtests reduces the occurrence. + // Supporting session would help us understand fix it https://github.com/FerretDB/FerretDB/issues/153. + // + //mongo.CommandError{ + // Code: 50738, + // Name: "Location50738", + // Message: "Cannot run getMore on cursor 5720627396082469624, which was created in session " + + // "95326129-ff9c-48a4-9060-464b4ea3ee06 - 47DEQpj8HBSa+/TImW+5JC\neuQeRkm5NMpJWZG3hSuFU= - - , " + + // "in session 9e8902e9-338c-4156-9fd8-50e5d62ac992 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - ", + //}, err = collection.Database().RunCommand(ctx, getMoreCommand).Decode(&res) if tc.err != nil { AssertEqualAltCommandError(t, *tc.err, tc.altMessage, err) @@ -1335,9 +1347,48 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { _, err := collection1.InsertMany(s.Ctx, docs) require.NoError(t, err) + t.Run("SameClient", func(t *testing.T) { + t.Parallel() + + var res bson.D + err = collection1.Database().RunCommand(s.Ctx, + bson.D{ + {"find", collection1.Name()}, + {"batchSize", 2}, + }).Decode(&res) + require.NoError(t, err) + + v, ok := res.Map()["cursor"] + require.True(t, ok) + + cursor, ok := v.(bson.D) + require.True(t, ok) + + cursorID := cursor.Map()["id"] + assert.NotNil(t, cursorID) + + // Even `find` and `getMore` use the same connection by setting `minPoolSize` and `maxPoolSize` to 1, + // on rare occasion it may return session error due to different session calling `find` and `getMore`. + // If that happens too frequently, removing t.Parallel() in subtests reduces the occurrence. + // Supporting session would help us understand fix it https://github.com/FerretDB/FerretDB/issues/153. + // + //mongo.CommandError{ + // Code: 50738, + // Name: "Location50738", + // Message: "Cannot run getMore on cursor 5720627396082469624, which was created in session " + + // "95326129-ff9c-48a4-9060-464b4ea3ee06 - 47DEQpj8HBSa+/TImW+5JC\neuQeRkm5NMpJWZG3hSuFU= - - , " + + // "in session 9e8902e9-338c-4156-9fd8-50e5d62ac992 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - ", + //}, + err = collection1.Database().RunCommand(s.Ctx, bson.D{ + {"getMore", cursorID}, + {"collection", collection1.Name()}, + }).Decode(&res) + require.NoError(t, err) + }) + t.Run("DifferentClient", func(t *testing.T) { // error returned from using different clients are session related error, - // currently FerretDB does not return an error + // hence currently FerretDB does not return an error setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/153") t.Parallel() @@ -1379,7 +1430,7 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { {"collection", collection2.Name()}, }).Decode(&res) - // use AssertEqualCommandError because message cannot be compared as it contains specific session ID + // use AssertEqualCommandError because message cannot be compared as it contains session ID AssertMatchesCommandError( t, mongo.CommandError{ From 805fcb2d348f6e5d255f79d961f9efc26bdf12aa Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 17:28:33 +0900 Subject: [PATCH 17/28] cleanup --- integration/commands_diagnostic_test.go | 5 ----- integration/query_test.go | 5 ----- integration/setup/client.go | 10 ++++++++++ integration/setup/listener.go | 8 ++++++-- integration/setup/setup.go | 6 +++--- integration/setup/setup_compat.go | 2 +- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 73e8a16d2eba..fe8ed629a16c 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -474,11 +474,6 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { u, err := url.Parse(s.MongoDBURI) require.NoError(t, err) - q2 := u.Query() - q2.Set("maxPoolSize", "1") - q2.Set("minPoolSize", "1") - u.RawQuery = q2.Encode() - client2, err := mongo.Connect(s.Ctx, options.Client().ApplyURI(u.String())) require.NoError(t, err) diff --git a/integration/query_test.go b/integration/query_test.go index c621460de888..e9334f9c1daa 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -1396,11 +1396,6 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { u, err := url.Parse(s.MongoDBURI) require.NoError(t, err) - q2 := u.Query() - q2.Set("maxPoolSize", "1") - q2.Set("minPoolSize", "1") - u.RawQuery = q2.Encode() - client2, err := mongo.Connect(s.Ctx, options.Client().ApplyURI(u.String())) require.NoError(t, err) diff --git a/integration/setup/client.go b/integration/setup/client.go index f0646c8b27fe..e3d8ac47f06e 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -36,6 +36,8 @@ type mongoDBURIOpts struct { hostPort string // for TCP and TLS unixSocketPath string tlsAndAuth bool + maxPoolSize string + minPoolSize string } // mongoDBURI builds MongoDB URI with given options. @@ -65,6 +67,14 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { user = url.UserPassword("username", "password") } + if opts.maxPoolSize != "" { + q.Set("maxPoolSize", opts.maxPoolSize) + } + + if opts.minPoolSize != "" { + q.Set("minPoolSize", opts.minPoolSize) + } + // TODO https://github.com/FerretDB/FerretDB/issues/1507 u := &url.URL{ Scheme: "mongodb", diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 2456440660ef..91c14d58faf2 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -17,6 +17,7 @@ package setup import ( "context" "errors" + "net/url" "os" "path/filepath" "strings" @@ -65,7 +66,7 @@ func unixSocketPath(tb testing.TB) string { // setupListener starts in-process FerretDB server that runs until ctx is canceled. // It returns client and MongoDB URI of that listener. -func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mongo.Client, string) { +func setupListener(tb testing.TB, ctx context.Context, extraOpts url.Values, logger *zap.Logger) (*mongo.Client, string) { tb.Helper() _, span := otel.Tracer("").Start(ctx, "setupListener") @@ -196,7 +197,10 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon <-runDone }) - var clientOpts mongoDBURIOpts + clientOpts := mongoDBURIOpts{ + maxPoolSize: extraOpts.Get("maxPoolSize"), + minPoolSize: extraOpts.Get("minPoolSize"), + } switch { case *targetTLSF: diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 6b28b173a228..a643f2c9a844 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -140,7 +140,7 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { var uri string if *targetURLF == "" { - client, uri = setupListener(tb, setupCtx, logger) + client, uri = setupListener(tb, setupCtx, opts.ExtraOptions, logger) } else { u, err := url.Parse(*targetURLF) require.NoError(tb, err) @@ -153,8 +153,8 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { } u.RawQuery = q.Encode() - client = setupClient(tb, setupCtx, u.String()) - uri = *targetURLF + uri = u.String() + client = setupClient(tb, setupCtx, uri) } // register cleanup function after setupListener registers its own to preserve full logs diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index f6edfaccc867..5cddac647dc3 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -94,7 +94,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul var targetClient *mongo.Client if *targetURLF == "" { - targetClient, _ = setupListener(tb, setupCtx, logger) + targetClient, _ = setupListener(tb, setupCtx, nil, logger) } else { targetClient = setupClient(tb, setupCtx, *targetURLF) } From 9e6b8e9869d526849ed702ba7cf54ffdbefb898c Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 18:01:21 +0900 Subject: [PATCH 18/28] do not run session related tests in parallel --- integration/query_test.go | 50 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index e9334f9c1daa..f337b14767b8 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -1225,7 +1225,17 @@ func TestQueryCommandGetMore(t *testing.T) { t.Skip(tc.skip) } - t.Parallel() + // Do not run subtests in t.Parallel() to reduce the occurrence + // of session error. Even the same connection is used for the test + // by setting `minPoolSize` and `maxPoolSize` to 1, on rare occasion + // it returns error due to different session calling `find` and `getMore`. + // Supporting session would help us understand fix it + // https://github.com/FerretDB/FerretDB/issues/153. + // + // > Location50738 + // > Cannot run getMore on cursor 2053655655200551971, + // > which was created in session 2926eea5-9775-41a3-a563-096969f1c7d5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - , + // > in session 774d9ac6-b24a-4fd8-9874-f92ab1c9c8f5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - require.NotNil(t, tc.firstBatch, "firstBatch must not be nil") @@ -1276,18 +1286,6 @@ func TestQueryCommandGetMore(t *testing.T) { getMoreRest..., ) - // Even `find` and `getMore` use the same connection by setting `minPoolSize` and `maxPoolSize` to 1, - // on rare occasion it may return session error due to different session calling `find` and `getMore`. - // If that happens too frequently, removing t.Parallel() in subtests reduces the occurrence. - // Supporting session would help us understand fix it https://github.com/FerretDB/FerretDB/issues/153. - // - //mongo.CommandError{ - // Code: 50738, - // Name: "Location50738", - // Message: "Cannot run getMore on cursor 5720627396082469624, which was created in session " + - // "95326129-ff9c-48a4-9060-464b4ea3ee06 - 47DEQpj8HBSa+/TImW+5JC\neuQeRkm5NMpJWZG3hSuFU= - - , " + - // "in session 9e8902e9-338c-4156-9fd8-50e5d62ac992 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - ", - //}, err = collection.Database().RunCommand(ctx, getMoreCommand).Decode(&res) if tc.err != nil { AssertEqualAltCommandError(t, *tc.err, tc.altMessage, err) @@ -1348,7 +1346,17 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { require.NoError(t, err) t.Run("SameClient", func(t *testing.T) { - t.Parallel() + // Do not run subtests in t.Parallel() to reduce the occurrence + // of session error. Even the same connection is used for the test + // by setting `minPoolSize` and `maxPoolSize` to 1, on rare occasion + // it returns error due to different session calling `find` and `getMore`. + // Supporting session would help us understand fix it + // https://github.com/FerretDB/FerretDB/issues/153. + // + // > Location50738 + // > Cannot run getMore on cursor 2053655655200551971, + // > which was created in session 2926eea5-9775-41a3-a563-096969f1c7d5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - , + // > in session 774d9ac6-b24a-4fd8-9874-f92ab1c9c8f5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - var res bson.D err = collection1.Database().RunCommand(s.Ctx, @@ -1367,18 +1375,6 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { cursorID := cursor.Map()["id"] assert.NotNil(t, cursorID) - // Even `find` and `getMore` use the same connection by setting `minPoolSize` and `maxPoolSize` to 1, - // on rare occasion it may return session error due to different session calling `find` and `getMore`. - // If that happens too frequently, removing t.Parallel() in subtests reduces the occurrence. - // Supporting session would help us understand fix it https://github.com/FerretDB/FerretDB/issues/153. - // - //mongo.CommandError{ - // Code: 50738, - // Name: "Location50738", - // Message: "Cannot run getMore on cursor 5720627396082469624, which was created in session " + - // "95326129-ff9c-48a4-9060-464b4ea3ee06 - 47DEQpj8HBSa+/TImW+5JC\neuQeRkm5NMpJWZG3hSuFU= - - , " + - // "in session 9e8902e9-338c-4156-9fd8-50e5d62ac992 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - ", - //}, err = collection1.Database().RunCommand(s.Ctx, bson.D{ {"getMore", cursorID}, {"collection", collection1.Name()}, @@ -1425,7 +1421,7 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { {"collection", collection2.Name()}, }).Decode(&res) - // use AssertEqualCommandError because message cannot be compared as it contains session ID + // use AssertMatchesCommandError because message cannot be compared as it contains session ID AssertMatchesCommandError( t, mongo.CommandError{ From 1e1f975f53662077fe05577fa0f2e0cf66d98d78 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 18:46:52 +0900 Subject: [PATCH 19/28] refactor --- integration/setup/client.go | 24 ++++++++++++++++++++---- integration/setup/setup.go | 17 +++++------------ integration/setup/test_helpers.go | 2 +- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index e3d8ac47f06e..d3f21e90fadd 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -33,6 +33,7 @@ import ( // mongoDBURIOpts represents mongoDBURI's options. type mongoDBURIOpts struct { + baseURI string hostPort string // for TCP and TLS unixSocketPath string tlsAndAuth bool @@ -45,16 +46,30 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { tb.Helper() var host string + q := make(url.Values) - if opts.hostPort != "" { - require.Empty(tb, opts.unixSocketPath, "both hostPort and unixSocketPath are set") + switch { + case opts.hostPort != "": + require.Empty(tb, opts.unixSocketPath, "only one of baseURI, hostPort or unixSocketPath should be set") + require.Empty(tb, opts.baseURI, "only one of baseURI, hostPort or unixSocketPath should be set") host = opts.hostPort - } else { + case opts.unixSocketPath != "": + require.Empty(tb, opts.hostPort, "only one of baseURI, hostPort or unixSocketPath should be set") + require.Empty(tb, opts.baseURI, "only one of baseURI, hostPort or unixSocketPath should be set") host = opts.unixSocketPath + case opts.baseURI != "": + require.Empty(tb, opts.unixSocketPath, "only one of baseURI, hostPort or unixSocketPath should be set") + require.Empty(tb, opts.hostPort, "only one of baseURI, hostPort or unixSocketPath should be set") + u, err := url.Parse(opts.baseURI) + require.NoError(tb, err) + + host = u.Host + q = u.Query() + default: + require.Empty(tb, opts.unixSocketPath, "one of baseURI, hostPort or unixSocketPath should be set") } var user *url.Userinfo - q := make(url.Values) if opts.tlsAndAuth { require.Empty(tb, opts.unixSocketPath, "unixSocketPath cannot be used with TLS") @@ -90,6 +105,7 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { // makeClient returns new client for the given working MongoDB URI. func makeClient(ctx context.Context, uri string) (*mongo.Client, error) { clientOpts := options.Client().ApplyURI(uri) + clientOpts.SetMonitor(otelmongo.NewMonitor()) client, err := mongo.Connect(ctx, clientOpts) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index a643f2c9a844..b62f67265226 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -142,18 +142,11 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { if *targetURLF == "" { client, uri = setupListener(tb, setupCtx, opts.ExtraOptions, logger) } else { - u, err := url.Parse(*targetURLF) - require.NoError(tb, err) - - q := u.Query() - for k, vs := range opts.ExtraOptions { - for _, v := range vs { - q.Set(k, v) - } - } - - u.RawQuery = q.Encode() - uri = u.String() + uri = mongoDBURI(tb, &mongoDBURIOpts{ + baseURI: *targetURLF, + maxPoolSize: opts.ExtraOptions.Get("maxPoolSize"), + minPoolSize: opts.ExtraOptions.Get("minPoolSize"), + }) client = setupClient(tb, setupCtx, uri) } diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index 1b152246543f..dce73c4b501d 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -42,7 +42,7 @@ func SkipExceptMongoDB(tb testing.TB, reason string) { if *targetBackendF != "mongodb" { require.NotEmpty(tb, reason, "reason must not be empty") - tb.Skipf("Skipping for MongoDB: %s.", reason) + tb.Skipf("Skipping for %s: %s.", *targetBackendF, reason) } } From fe31dfdd4a29d76493e0cf38ce482c67f93e5a13 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 27 Jun 2023 18:49:26 +0900 Subject: [PATCH 20/28] fix --- integration/setup/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index d3f21e90fadd..7b01e67a61aa 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -66,7 +66,7 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { host = u.Host q = u.Query() default: - require.Empty(tb, opts.unixSocketPath, "one of baseURI, hostPort or unixSocketPath should be set") + tb.Fatal("one of baseURI, hostPort or unixSocketPath should be set") } var user *url.Userinfo From aae64c808a20eec917ab4242a536d553af534458 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 28 Jun 2023 16:35:44 +0900 Subject: [PATCH 21/28] separate mongodburi and setup listener from extra options --- integration/setup/client.go | 33 ++++--------------------------- integration/setup/listener.go | 8 ++------ integration/setup/setup.go | 29 +++++++++++++++++++-------- integration/setup/setup_compat.go | 2 +- 4 files changed, 28 insertions(+), 44 deletions(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index 7b01e67a61aa..91db78793b08 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -33,12 +33,9 @@ import ( // mongoDBURIOpts represents mongoDBURI's options. type mongoDBURIOpts struct { - baseURI string hostPort string // for TCP and TLS unixSocketPath string tlsAndAuth bool - maxPoolSize string - minPoolSize string } // mongoDBURI builds MongoDB URI with given options. @@ -46,30 +43,16 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { tb.Helper() var host string - q := make(url.Values) - switch { - case opts.hostPort != "": - require.Empty(tb, opts.unixSocketPath, "only one of baseURI, hostPort or unixSocketPath should be set") - require.Empty(tb, opts.baseURI, "only one of baseURI, hostPort or unixSocketPath should be set") + if opts.hostPort != "" { + require.Empty(tb, opts.unixSocketPath, "both hostPort and unixSocketPath are set") host = opts.hostPort - case opts.unixSocketPath != "": - require.Empty(tb, opts.hostPort, "only one of baseURI, hostPort or unixSocketPath should be set") - require.Empty(tb, opts.baseURI, "only one of baseURI, hostPort or unixSocketPath should be set") + } else { host = opts.unixSocketPath - case opts.baseURI != "": - require.Empty(tb, opts.unixSocketPath, "only one of baseURI, hostPort or unixSocketPath should be set") - require.Empty(tb, opts.hostPort, "only one of baseURI, hostPort or unixSocketPath should be set") - u, err := url.Parse(opts.baseURI) - require.NoError(tb, err) - - host = u.Host - q = u.Query() - default: - tb.Fatal("one of baseURI, hostPort or unixSocketPath should be set") } var user *url.Userinfo + q := make(url.Values) if opts.tlsAndAuth { require.Empty(tb, opts.unixSocketPath, "unixSocketPath cannot be used with TLS") @@ -82,14 +65,6 @@ func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { user = url.UserPassword("username", "password") } - if opts.maxPoolSize != "" { - q.Set("maxPoolSize", opts.maxPoolSize) - } - - if opts.minPoolSize != "" { - q.Set("minPoolSize", opts.minPoolSize) - } - // TODO https://github.com/FerretDB/FerretDB/issues/1507 u := &url.URL{ Scheme: "mongodb", diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 91c14d58faf2..2456440660ef 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -17,7 +17,6 @@ package setup import ( "context" "errors" - "net/url" "os" "path/filepath" "strings" @@ -66,7 +65,7 @@ func unixSocketPath(tb testing.TB) string { // setupListener starts in-process FerretDB server that runs until ctx is canceled. // It returns client and MongoDB URI of that listener. -func setupListener(tb testing.TB, ctx context.Context, extraOpts url.Values, logger *zap.Logger) (*mongo.Client, string) { +func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mongo.Client, string) { tb.Helper() _, span := otel.Tracer("").Start(ctx, "setupListener") @@ -197,10 +196,7 @@ func setupListener(tb testing.TB, ctx context.Context, extraOpts url.Values, log <-runDone }) - clientOpts := mongoDBURIOpts{ - maxPoolSize: extraOpts.Get("maxPoolSize"), - minPoolSize: extraOpts.Get("minPoolSize"), - } + var clientOpts mongoDBURIOpts switch { case *targetTLSF: diff --git a/integration/setup/setup.go b/integration/setup/setup.go index b62f67265226..513e07d60065 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -137,16 +137,29 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { logger := testutil.LevelLogger(tb, level) var client *mongo.Client - var uri string + uri := *targetURLF if *targetURLF == "" { - client, uri = setupListener(tb, setupCtx, opts.ExtraOptions, logger) - } else { - uri = mongoDBURI(tb, &mongoDBURIOpts{ - baseURI: *targetURLF, - maxPoolSize: opts.ExtraOptions.Get("maxPoolSize"), - minPoolSize: opts.ExtraOptions.Get("minPoolSize"), - }) + client, uri = setupListener(tb, setupCtx, logger) + } + + if opts.ExtraOptions != nil || *targetURLF != "" { + u, err := url.Parse(uri) + require.NoError(tb, err) + + q := u.Query() + for k, vs := range opts.ExtraOptions { + for _, v := range vs { + q.Set(k, v) + } + } + + u.RawQuery = q.Encode() + uri = u.String() + + // If ExtraOptions is set for in-process FerretDB, two clients are created. + // setupListener creates a client to check in-process FerretDB, + // and this client used for tests with extra options. client = setupClient(tb, setupCtx, uri) } diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index 5cddac647dc3..f6edfaccc867 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -94,7 +94,7 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul var targetClient *mongo.Client if *targetURLF == "" { - targetClient, _ = setupListener(tb, setupCtx, nil, logger) + targetClient, _ = setupListener(tb, setupCtx, logger) } else { targetClient = setupClient(tb, setupCtx, *targetURLF) } From a22d37c88b4510887faae8aaa3b2624037c29cbc Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 28 Jun 2023 16:48:56 +0900 Subject: [PATCH 22/28] refactoring func name --- integration/setup/client.go | 8 ++++---- integration/setup/listener.go | 12 ++++++------ integration/setup/setup.go | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index 91db78793b08..3c7220c773dd 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -31,15 +31,15 @@ import ( "github.com/FerretDB/FerretDB/internal/util/observability" ) -// mongoDBURIOpts represents mongoDBURI's options. -type mongoDBURIOpts struct { +// buildURIForInProcessFerretDBOpts represents buildURIForInProcessFerretDB's options. +type buildURIForInProcessFerretDBOpts struct { hostPort string // for TCP and TLS unixSocketPath string tlsAndAuth bool } -// mongoDBURI builds MongoDB URI with given options. -func mongoDBURI(tb testing.TB, opts *mongoDBURIOpts) string { +// buildURIForInProcessFerretDB is used to construct URI from options obtained from in-process FerretDB. +func buildURIForInProcessFerretDB(tb testing.TB, opts *buildURIForInProcessFerretDBOpts) string { tb.Helper() var host string diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 2456440660ef..f8d1be7eb8bc 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -196,21 +196,21 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon <-runDone }) - var clientOpts mongoDBURIOpts + var opts buildURIForInProcessFerretDBOpts switch { case *targetTLSF: - clientOpts.hostPort = l.TLSAddr().String() - clientOpts.tlsAndAuth = true + opts.hostPort = l.TLSAddr().String() + opts.tlsAndAuth = true case *targetUnixSocketF: - clientOpts.unixSocketPath = l.UnixAddr().String() + opts.unixSocketPath = l.UnixAddr().String() default: - clientOpts.hostPort = l.TCPAddr().String() + opts.hostPort = l.TCPAddr().String() } // those will fail the test if in-process FerretDB is not working; // for example, when backend is down - uri := mongoDBURI(tb, &clientOpts) + uri := buildURIForInProcessFerretDB(tb, &opts) client := setupClient(tb, ctx, uri) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 513e07d60065..a099c650d650 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -158,7 +158,7 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { uri = u.String() // If ExtraOptions is set for in-process FerretDB, two clients are created. - // setupListener creates a client to check in-process FerretDB, + // setupListener creates a client to check in-process FerretDB parameters, // and this client used for tests with extra options. client = setupClient(tb, setupCtx, uri) } From 1a65aac4b538a7cfcf0260ca1404298360cd2622 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 28 Jun 2023 18:13:28 +0900 Subject: [PATCH 23/28] update comments and rename --- integration/commands_diagnostic_test.go | 22 +++----- integration/query_test.go | 75 ++++++++++++++----------- integration/setup/setup.go | 2 +- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index fe8ed629a16c..b9374e9d43a6 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -407,19 +407,17 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { } } -// TestCommandWhatsMyURIConnection tests that a client uses the same connection for -// all commands run by the client, and different clients use different connection. -// The port number is used to validate if the same connection is used or not, -// since unique port number is assigned to each client connection in the test setup. +// TestCommandWhatsMyURIConnection tests that integration test setup applies +// minPoolSize, maxPoolSize and maxIdleTimeMS correctly to the driver. func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() - // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, - // which forces a client to use a single connection pool + // single connection is used by the client created from setup s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ - "minPoolSize": []string{"1"}, - "maxPoolSize": []string{"1"}, + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + "maxIdleTimeMS": []string{"0"}, }, }) @@ -458,13 +456,9 @@ func TestCommandWhatsMyURIConnection(t *testing.T) { close(ports) - var allPorts []string + firstPort := <-ports for port := range ports { - allPorts = append(allPorts, port) - } - - for _, port := range allPorts { - require.Equal(t, allPorts[0], port, "expected same client uses the same port: %s", allPorts) + require.Equal(t, firstPort, port, "expected same client to use the same port") } }) diff --git a/integration/query_test.go b/integration/query_test.go index f337b14767b8..51bf9b235e69 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -958,12 +958,12 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() - // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, - // which forces a client to use a single connection pool + // single connection is used by the client created from setup s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ - "minPoolSize": []string{"1"}, - "maxPoolSize": []string{"1"}, + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + "maxIdleTimeMS": []string{"0"}, }, }) @@ -1225,10 +1225,8 @@ func TestQueryCommandGetMore(t *testing.T) { t.Skip(tc.skip) } - // Do not run subtests in t.Parallel() to reduce the occurrence - // of session error. Even the same connection is used for the test - // by setting `minPoolSize` and `maxPoolSize` to 1, on rare occasion - // it returns error due to different session calling `find` and `getMore`. + // Do not run subtests in t.Parallel() to eliminate the occurrence + // of session error. // Supporting session would help us understand fix it // https://github.com/FerretDB/FerretDB/issues/153. // @@ -1328,28 +1326,27 @@ func TestQueryCommandGetMore(t *testing.T) { func TestQueryCommandGetMoreConnection(t *testing.T) { t.Parallel() - // set minPoolSize and maxPoolSize 1 to ensure only one pool exists duration of the test, - // which forces a client to use a single connection pool + // single connection is used by the client created from setup s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ - "minPoolSize": []string{"1"}, - "maxPoolSize": []string{"1"}, + "minPoolSize": []string{"1"}, + "maxPoolSize": []string{"1"}, + "maxIdleTimeMS": []string{"0"}, }, }) + ctx := s.Ctx collection1 := s.Collection databaseName := s.Collection.Database().Name() collectionName := s.Collection.Name() docs := generateDocuments(0, 5) - _, err := collection1.InsertMany(s.Ctx, docs) + _, err := collection1.InsertMany(ctx, docs) require.NoError(t, err) t.Run("SameClient", func(t *testing.T) { - // Do not run subtests in t.Parallel() to reduce the occurrence - // of session error. Even the same connection is used for the test - // by setting `minPoolSize` and `maxPoolSize` to 1, on rare occasion - // it returns error due to different session calling `find` and `getMore`. + // Do not run subtests in t.Parallel() to eliminate the occurrence + // of session error. // Supporting session would help us understand fix it // https://github.com/FerretDB/FerretDB/issues/153. // @@ -1359,11 +1356,13 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { // > in session 774d9ac6-b24a-4fd8-9874-f92ab1c9c8f5 - 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= - - var res bson.D - err = collection1.Database().RunCommand(s.Ctx, + err = collection1.Database().RunCommand( + ctx, bson.D{ {"find", collection1.Name()}, {"batchSize", 2}, - }).Decode(&res) + }, + ).Decode(&res) require.NoError(t, err) v, ok := res.Map()["cursor"] @@ -1375,36 +1374,41 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { cursorID := cursor.Map()["id"] assert.NotNil(t, cursorID) - err = collection1.Database().RunCommand(s.Ctx, bson.D{ - {"getMore", cursorID}, - {"collection", collection1.Name()}, - }).Decode(&res) + err = collection1.Database().RunCommand( + ctx, + bson.D{ + {"getMore", cursorID}, + {"collection", collection1.Name()}, + }, + ).Decode(&res) require.NoError(t, err) }) t.Run("DifferentClient", func(t *testing.T) { - // error returned from using different clients are session related error, - // hence currently FerretDB does not return an error + // The error returned from MongoDB is a session error, FerretDB does not + // return an error because db, collection and username are the same. setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/153") - t.Parallel() + // Do not run subtest in t.Parallel() to avoid breaking when another subtest is added. u, err := url.Parse(s.MongoDBURI) require.NoError(t, err) - client2, err := mongo.Connect(s.Ctx, options.Client().ApplyURI(u.String())) + client2, err := mongo.Connect(ctx, options.Client().ApplyURI(u.String())) require.NoError(t, err) - defer client2.Disconnect(s.Ctx) + defer client2.Disconnect(ctx) collection2 := client2.Database(databaseName).Collection(collectionName) var res bson.D - err = collection1.Database().RunCommand(s.Ctx, + err = collection1.Database().RunCommand( + ctx, bson.D{ {"find", collection1.Name()}, {"batchSize", 2}, - }).Decode(&res) + }, + ).Decode(&res) require.NoError(t, err) v, ok := res.Map()["cursor"] @@ -1416,10 +1420,13 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { cursorID := cursor.Map()["id"] assert.NotNil(t, cursorID) - err = collection2.Database().RunCommand(s.Ctx, bson.D{ - {"getMore", cursorID}, - {"collection", collection2.Name()}, - }).Decode(&res) + err = collection2.Database().RunCommand( + ctx, + bson.D{ + {"getMore", cursorID}, + {"collection", collection2.Name()}, + }, + ).Decode(&res) // use AssertMatchesCommandError because message cannot be compared as it contains session ID AssertMatchesCommandError( diff --git a/integration/setup/setup.go b/integration/setup/setup.go index a099c650d650..e57a083c6f51 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -88,7 +88,7 @@ type SetupOpts struct { // Benchmark data provider. If empty, collection is not created. BenchmarkProvider shareddata.BenchmarkProvider - // ExtraOptions overwrites the options set in MongoDB URI when the same option is set. + // ExtraOptions sets the options in MongoDB URI, when the option exists it overwrites that option. ExtraOptions url.Values } From 0e7adb8ccae8409c1ec848a23c17cf251fa2bddd Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 28 Jun 2023 18:31:40 +0900 Subject: [PATCH 24/28] cleanup --- integration/commands_diagnostic_test.go | 2 +- integration/query_test.go | 6 +++--- integration/setup/client.go | 8 ++++---- integration/setup/listener.go | 4 ++-- integration/setup/setup.go | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index b9374e9d43a6..3957c7909c64 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -412,7 +412,7 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() - // single connection is used by the client created from setup + // options are applied to create a client that uses single connection pool s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ "minPoolSize": []string{"1"}, diff --git a/integration/query_test.go b/integration/query_test.go index 51bf9b235e69..d116f4a7b4fc 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -958,7 +958,7 @@ func TestQueryBatchSize(t *testing.T) { func TestQueryCommandGetMore(t *testing.T) { t.Parallel() - // single connection is used by the client created from setup + // options are applied to create a client that uses single connection pool s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ "minPoolSize": []string{"1"}, @@ -1326,7 +1326,7 @@ func TestQueryCommandGetMore(t *testing.T) { func TestQueryCommandGetMoreConnection(t *testing.T) { t.Parallel() - // single connection is used by the client created from setup + // options are applied to create a client that uses single connection pool s := setup.SetupWithOpts(t, &setup.SetupOpts{ ExtraOptions: url.Values{ "minPoolSize": []string{"1"}, @@ -1389,7 +1389,7 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { // return an error because db, collection and username are the same. setup.SkipExceptMongoDB(t, "https://github.com/FerretDB/FerretDB/issues/153") - // Do not run subtest in t.Parallel() to avoid breaking when another subtest is added. + // do not run subtest in parallel to avoid breaking another parallel subtest u, err := url.Parse(s.MongoDBURI) require.NoError(t, err) diff --git a/integration/setup/client.go b/integration/setup/client.go index 3c7220c773dd..faea3e18c59b 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -31,15 +31,15 @@ import ( "github.com/FerretDB/FerretDB/internal/util/observability" ) -// buildURIForInProcessFerretDBOpts represents buildURIForInProcessFerretDB's options. -type buildURIForInProcessFerretDBOpts struct { +// buildInProcessFerretDBURIOpts represents buildInProcessFerretDBURI's options. +type buildInProcessFerretDBURIOpts struct { hostPort string // for TCP and TLS unixSocketPath string tlsAndAuth bool } -// buildURIForInProcessFerretDB is used to construct URI from options obtained from in-process FerretDB. -func buildURIForInProcessFerretDB(tb testing.TB, opts *buildURIForInProcessFerretDBOpts) string { +// buildInProcessFerretDBURI is used to construct URI from options obtained from in-process FerretDB. +func buildInProcessFerretDBURI(tb testing.TB, opts *buildInProcessFerretDBURIOpts) string { tb.Helper() var host string diff --git a/integration/setup/listener.go b/integration/setup/listener.go index f8d1be7eb8bc..fde71dff90c1 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -196,7 +196,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon <-runDone }) - var opts buildURIForInProcessFerretDBOpts + var opts buildInProcessFerretDBURIOpts switch { case *targetTLSF: @@ -210,7 +210,7 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon // those will fail the test if in-process FerretDB is not working; // for example, when backend is down - uri := buildURIForInProcessFerretDB(tb, &opts) + uri := buildInProcessFerretDBURI(tb, &opts) client := setupClient(tb, ctx, uri) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index e57a083c6f51..01fa6ffa1a92 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -157,9 +157,9 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { u.RawQuery = q.Encode() uri = u.String() - // If ExtraOptions is set for in-process FerretDB, two clients are created. - // setupListener creates a client to check in-process FerretDB parameters, - // and this client used for tests with extra options. + // If ExtraOptions is set for in-process FerretDB, two clients are created + // on purpose. setupListener creates a client to check in-process FerretDB + // options, and this client is created with extra options used for tests. client = setupClient(tb, setupCtx, uri) } From c2397d2c880de1e5ee2b03c4b70fba90c2039c44 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 28 Jun 2023 19:58:48 +0900 Subject: [PATCH 25/28] fix new linter --- integration/query_test.go | 21 +++++++++++++-------- integration/setup/setup.go | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index d116f4a7b4fc..daafc5d63d8f 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -30,6 +30,7 @@ import ( "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" + "github.com/FerretDB/FerretDB/internal/types" ) func TestQueryBadFindType(t *testing.T) { @@ -1365,13 +1366,15 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { ).Decode(&res) require.NoError(t, err) - v, ok := res.Map()["cursor"] - require.True(t, ok) + doc := ConvertDocument(t, res) + + v, _ := doc.Get("cursor") + require.NotNil(t, v) - cursor, ok := v.(bson.D) + cursor, ok := v.(*types.Document) require.True(t, ok) - cursorID := cursor.Map()["id"] + cursorID, _ := cursor.Get("id") assert.NotNil(t, cursorID) err = collection1.Database().RunCommand( @@ -1411,13 +1414,15 @@ func TestQueryCommandGetMoreConnection(t *testing.T) { ).Decode(&res) require.NoError(t, err) - v, ok := res.Map()["cursor"] - require.True(t, ok) + doc := ConvertDocument(t, res) + + v, _ := doc.Get("cursor") + require.NotNil(t, v) - cursor, ok := v.(bson.D) + cursor, ok := v.(*types.Document) require.True(t, ok) - cursorID := cursor.Map()["id"] + cursorID, _ := cursor.Get("id") assert.NotNil(t, cursorID) err = collection2.Database().RunCommand( diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 01fa6ffa1a92..953b3f9eef2b 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -148,6 +148,7 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { require.NoError(tb, err) q := u.Query() + for k, vs := range opts.ExtraOptions { for _, v := range vs { q.Set(k, v) From d5df4e013837af622e456128815c4a3284442c7f Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 28 Jun 2023 22:09:12 +0400 Subject: [PATCH 26/28] Tweak comment --- integration/commands_diagnostic_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/commands_diagnostic_test.go b/integration/commands_diagnostic_test.go index 3957c7909c64..b24a0242cf90 100644 --- a/integration/commands_diagnostic_test.go +++ b/integration/commands_diagnostic_test.go @@ -409,6 +409,7 @@ func TestCommandsDiagnosticWhatsMyURI(t *testing.T) { // TestCommandWhatsMyURIConnection tests that integration test setup applies // minPoolSize, maxPoolSize and maxIdleTimeMS correctly to the driver. +// It also tests that the driver behaves like we think it should. func TestCommandWhatsMyURIConnection(t *testing.T) { t.Parallel() From 17db130f810e91322a1fc7141c8bcaffce159c20 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 28 Jun 2023 22:27:56 +0400 Subject: [PATCH 27/28] Simplify --- integration/setup/client.go | 48 -------------------- integration/setup/listener.go | 73 +++++++++++++++++++++---------- integration/setup/setup.go | 16 +++---- integration/setup/setup_compat.go | 3 +- 4 files changed, 59 insertions(+), 81 deletions(-) diff --git a/integration/setup/client.go b/integration/setup/client.go index faea3e18c59b..82d1bb58b21d 100644 --- a/integration/setup/client.go +++ b/integration/setup/client.go @@ -16,8 +16,6 @@ package setup import ( "context" - "net/url" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -31,52 +29,6 @@ import ( "github.com/FerretDB/FerretDB/internal/util/observability" ) -// buildInProcessFerretDBURIOpts represents buildInProcessFerretDBURI's options. -type buildInProcessFerretDBURIOpts struct { - hostPort string // for TCP and TLS - unixSocketPath string - tlsAndAuth bool -} - -// buildInProcessFerretDBURI is used to construct URI from options obtained from in-process FerretDB. -func buildInProcessFerretDBURI(tb testing.TB, opts *buildInProcessFerretDBURIOpts) string { - tb.Helper() - - var host string - - if opts.hostPort != "" { - require.Empty(tb, opts.unixSocketPath, "both hostPort and unixSocketPath are set") - host = opts.hostPort - } else { - host = opts.unixSocketPath - } - - var user *url.Userinfo - q := make(url.Values) - - if opts.tlsAndAuth { - require.Empty(tb, opts.unixSocketPath, "unixSocketPath cannot be used with TLS") - - // we don't separate TLS and auth just for simplicity of our test configurations - q.Set("tls", "true") - q.Set("tlsCertificateKeyFile", filepath.Join(CertsRoot, "client.pem")) - q.Set("tlsCaFile", filepath.Join(CertsRoot, "rootCA-cert.pem")) - q.Set("authMechanism", "PLAIN") - user = url.UserPassword("username", "password") - } - - // TODO https://github.com/FerretDB/FerretDB/issues/1507 - u := &url.URL{ - Scheme: "mongodb", - Host: host, - Path: "/", - User: user, - RawQuery: q.Encode(), - } - - return u.String() -} - // makeClient returns new client for the given working MongoDB URI. func makeClient(ctx context.Context, uri string) (*mongo.Client, error) { clientOpts := options.Client().ApplyURI(uri) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index fde71dff90c1..7f33f17d5cdd 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -17,6 +17,7 @@ package setup import ( "context" "errors" + "net/url" "os" "path/filepath" "strings" @@ -24,7 +25,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/mongo" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -63,9 +63,48 @@ func unixSocketPath(tb testing.TB) string { return f.Name() } +// listenerMongoDBURI builds MongoDB URI for in-process FerretDB. +func listenerMongoDBURI(tb testing.TB, hostPort, unixSocketPath string, tlsAndAuth bool) string { + tb.Helper() + + var host string + + if hostPort != "" { + require.Empty(tb, unixSocketPath, "both hostPort and unixSocketPath are set") + host = hostPort + } else { + host = unixSocketPath + } + + var user *url.Userinfo + q := make(url.Values) + + if tlsAndAuth { + require.Empty(tb, unixSocketPath, "unixSocketPath cannot be used with TLS") + + // we don't separate TLS and auth just for simplicity of our test configurations + q.Set("tls", "true") + q.Set("tlsCertificateKeyFile", filepath.Join(CertsRoot, "client.pem")) + q.Set("tlsCaFile", filepath.Join(CertsRoot, "rootCA-cert.pem")) + q.Set("authMechanism", "PLAIN") + user = url.UserPassword("username", "password") + } + + // TODO https://github.com/FerretDB/FerretDB/issues/1507 + u := &url.URL{ + Scheme: "mongodb", + Host: host, + Path: "/", + User: user, + RawQuery: q.Encode(), + } + + return u.String() +} + // setupListener starts in-process FerretDB server that runs until ctx is canceled. -// It returns client and MongoDB URI of that listener. -func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mongo.Client, string) { +// It returns basic MongoDB URI for that listener. +func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) string { tb.Helper() _, span := otel.Tracer("").Start(ctx, "setupListener") @@ -170,20 +209,12 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon l := clientconn.NewListener(&listenerOpts) - runCtx, runCancel := context.WithCancel(ctx) runDone := make(chan struct{}) - // that prevents the deadlock on failed client setup; see below - defer func() { - if tb.Failed() { - runCancel() - } - }() - go func() { defer close(runDone) - err := l.Run(runCtx) + err := l.Run(ctx) if err == nil || errors.Is(err, context.Canceled) { logger.Info("Listener stopped without error") } else { @@ -196,24 +227,22 @@ func setupListener(tb testing.TB, ctx context.Context, logger *zap.Logger) (*mon <-runDone }) - var opts buildInProcessFerretDBURIOpts + var hostPort, unixSocketPath string + var tlsAndAuth bool switch { case *targetTLSF: - opts.hostPort = l.TLSAddr().String() - opts.tlsAndAuth = true + hostPort = l.TLSAddr().String() + tlsAndAuth = true case *targetUnixSocketF: - opts.unixSocketPath = l.UnixAddr().String() + unixSocketPath = l.UnixAddr().String() default: - opts.hostPort = l.TCPAddr().String() + hostPort = l.TCPAddr().String() } - // those will fail the test if in-process FerretDB is not working; - // for example, when backend is down - uri := buildInProcessFerretDBURI(tb, &opts) - client := setupClient(tb, ctx, uri) + uri := listenerMongoDBURI(tb, hostPort, unixSocketPath, tlsAndAuth) logger.Info("Listener started", zap.String("handler", handler), zap.String("uri", uri)) - return client, uri + return uri } diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 953b3f9eef2b..b4820cc9b26e 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -136,14 +136,12 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { } logger := testutil.LevelLogger(tb, level) - var client *mongo.Client uri := *targetURLF - - if *targetURLF == "" { - client, uri = setupListener(tb, setupCtx, logger) + if uri == "" { + uri = setupListener(tb, setupCtx, logger) } - if opts.ExtraOptions != nil || *targetURLF != "" { + if opts.ExtraOptions != nil { u, err := url.Parse(uri) require.NoError(tb, err) @@ -157,13 +155,11 @@ func SetupWithOpts(tb testing.TB, opts *SetupOpts) *SetupResult { u.RawQuery = q.Encode() uri = u.String() - - // If ExtraOptions is set for in-process FerretDB, two clients are created - // on purpose. setupListener creates a client to check in-process FerretDB - // options, and this client is created with extra options used for tests. - client = setupClient(tb, setupCtx, uri) + tb.Logf("URI with extra options: %s", uri) } + client := setupClient(tb, setupCtx, uri) + // register cleanup function after setupListener registers its own to preserve full logs tb.Cleanup(cancel) diff --git a/integration/setup/setup_compat.go b/integration/setup/setup_compat.go index f6edfaccc867..b4e4278e37d1 100644 --- a/integration/setup/setup_compat.go +++ b/integration/setup/setup_compat.go @@ -94,7 +94,8 @@ func SetupCompatWithOpts(tb testing.TB, opts *SetupCompatOpts) *SetupCompatResul var targetClient *mongo.Client if *targetURLF == "" { - targetClient, _ = setupListener(tb, setupCtx, logger) + uri := setupListener(tb, setupCtx, logger) + targetClient = setupClient(tb, setupCtx, uri) } else { targetClient = setupClient(tb, setupCtx, *targetURLF) } From 676af8abd7a8de28ecafd918caae2ba3aeaa9a14 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 28 Jun 2023 22:33:10 +0400 Subject: [PATCH 28/28] Refactor --- ferretdb/ferretdb.go | 6 +++--- integration/setup/listener.go | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ferretdb/ferretdb.go b/ferretdb/ferretdb.go index e3faee3589c4..43c4cdd70693 100644 --- a/ferretdb/ferretdb.go +++ b/ferretdb/ferretdb.go @@ -184,9 +184,9 @@ func (f *FerretDB) MongoDBURI() string { switch { case f.config.Listener.TLS != "": - q := make(url.Values) - - q.Set("tls", "true") + q := url.Values{ + "tls": []string{"true"}, + } u = &url.URL{ Scheme: "mongodb", diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 7f33f17d5cdd..28fe876b3159 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -77,16 +77,18 @@ func listenerMongoDBURI(tb testing.TB, hostPort, unixSocketPath string, tlsAndAu } var user *url.Userinfo - q := make(url.Values) + var q url.Values if tlsAndAuth { require.Empty(tb, unixSocketPath, "unixSocketPath cannot be used with TLS") // we don't separate TLS and auth just for simplicity of our test configurations - q.Set("tls", "true") - q.Set("tlsCertificateKeyFile", filepath.Join(CertsRoot, "client.pem")) - q.Set("tlsCaFile", filepath.Join(CertsRoot, "rootCA-cert.pem")) - q.Set("authMechanism", "PLAIN") + q = url.Values{ + "tls": []string{"true"}, + "tlsCertificateKeyFile": []string{filepath.Join(CertsRoot, "client.pem")}, + "tlsCaFile": []string{filepath.Join(CertsRoot, "rootCA-cert.pem")}, + "authMechanism": []string{"PLAIN"}, + } user = url.UserPassword("username", "password") }