From c7fb0457a7094b5ab4ab9179d820050769b49c38 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 13:51:09 +0100 Subject: [PATCH 01/25] wip --- integration/Taskfile.yml | 2 +- integration/aggregate_documents_compat_test.go | 2 +- integration/query_command_compat_test.go | 2 +- integration/query_compat_test.go | 2 +- integration/query_test.go | 4 ++-- internal/backends/collection.go | 2 +- internal/backends/collection_test.go | 2 +- internal/backends/postgresql/collection.go | 2 +- internal/backends/sqlite/collection.go | 2 +- internal/handlers/sqlite/msg_explain.go | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration/Taskfile.yml b/integration/Taskfile.yml index 0550a7c6af97..3c79f0e03e91 100644 --- a/integration/Taskfile.yml +++ b/integration/Taskfile.yml @@ -93,7 +93,7 @@ tasks: - ../bin/benchstat{{exeExt}} old-sqlite.txt new-sqlite.txt bench-sqlite-no-pushdown: - desc: "Run benchmarks for `sqlite` with query pushdown disabled" + desc: "Run benchmarks for `sqlite` with filter pushdown disabled" cmds: - > go test -tags={{.BUILD_TAGS}} -timeout=0 -run=XXX diff --git a/integration/aggregate_documents_compat_test.go b/integration/aggregate_documents_compat_test.go index 34a63d9eda10..4d813c04f3de 100644 --- a/integration/aggregate_documents_compat_test.go +++ b/integration/aggregate_documents_compat_test.go @@ -117,7 +117,7 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P // TODO https://github.com/FerretDB/FerretDB/issues/3386 if setup.FilterPushdownDisabled() { resultPushdown = noPushdown - msg = "Query pushdown is disabled, but target resulted with pushdown" + msg = "Fitler pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, explainRes) diff --git a/integration/query_command_compat_test.go b/integration/query_command_compat_test.go index 0225623e6095..c2ffdf0f5e63 100644 --- a/integration/query_command_compat_test.go +++ b/integration/query_command_compat_test.go @@ -104,7 +104,7 @@ func testQueryCommandCompat(t *testing.T, testCases map[string]queryCommandCompa var msg string if setup.FilterPushdownDisabled() { tc.resultPushdown = noPushdown - msg = "Query pushdown is disabled, but target resulted with pushdown" + msg = "Filter pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, explainRes) diff --git a/integration/query_compat_test.go b/integration/query_compat_test.go index 33c1e2c7ac31..6eb843a0c717 100644 --- a/integration/query_compat_test.go +++ b/integration/query_compat_test.go @@ -133,7 +133,7 @@ func testQueryCompatWithProviders(t *testing.T, providers shareddata.Providers, var msg string if setup.FilterPushdownDisabled() { resultPushdown = noPushdown - msg = "Query pushdown is disabled, but target resulted with pushdown" + msg = "Filter pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, explainRes) diff --git a/integration/query_test.go b/integration/query_test.go index 4542003b1442..9a7b1d975226 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -824,7 +824,7 @@ func TestQueryCommandSingleBatch(t *testing.T) { func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { t.Parallel() - // must use a collection of documents which does not support query pushdown to test limit pushdown + // must use a collection of documents which does not support filter pushdown to test limit pushdown s := setup.SetupWithOpts(t, &setup.SetupOpts{Providers: []shareddata.Provider{shareddata.Composites}}) ctx, collection := s.Ctx, s.Collection @@ -1004,7 +1004,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { if setup.FilterPushdownDisabled() { resultPushdown = noPushdown - msg = "Query pushdown is disabled, but target resulted with pushdown" + msg = "Filter pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, res) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index e7be7aab494f..3a8ebbbb11c5 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -214,7 +214,7 @@ type ExplainParams struct { // ExplainResult represents the results of Collection.Explain method. type ExplainResult struct { QueryPlanner *types.Document - QueryPushdown bool + FilterPushdown bool UnsafeSortPushdown bool UnsafeLimitPushdown bool } diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index 68a224381f40..29bbc6ef930b 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -173,7 +173,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, &backends.ExplainParams{Filter: filter}) require.NoError(t, err) - assert.True(t, explainRes.QueryPushdown) + assert.True(t, explainRes.FilterPushdown) assert.True(t, explainRes.UnsafeSortPushdown) }) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index fdcdbd6928bf..b3ddb808747c 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -306,7 +306,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - res.QueryPushdown = where != "" + res.FilterPushdown = where != "" q += where diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 846c8eee9687..23b5de144a78 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -315,7 +315,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return &backends.ExplainResult{ QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), - QueryPushdown: queryPushdown, + FilterPushdown: queryPushdown, UnsafeSortPushdown: unsafeSortPushdown, UnsafeLimitPushdown: unsafeLimitPushdown, }, nil diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 775b365338c9..7c15810ca5b3 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -137,7 +137,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // our extensions // TODO https://github.com/FerretDB/FerretDB/issues/3235 - "pushdown", res.QueryPushdown, + "pushdown", res.FilterPushdown, "sortingPushdown", res.UnsafeSortPushdown, "limitPushdown", res.UnsafeLimitPushdown, From 2877a31fe07d1cab8d506146d3f765b36b96707d Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 13:53:24 +0100 Subject: [PATCH 02/25] wip --- integration/query_test.go | 26 +++++++++++++------------- internal/backends/sqlite/collection.go | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 9a7b1d975226..52ffb4872cab 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -835,7 +835,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { optSkip *int64 // optional, nil to leave optSkip unset len int // expected length of results - queryPushdown resultPushdown // optional, defaults to noPushdown + filterPushdown resultPushdown // optional, defaults to noPushdown unsafeLimitPushdown bool // optional, set true for expected pushdown for limit err *mongo.CommandError // optional, expected error from MongoDB altMessage string // optional, alternative error message for FerretDB, ignored if empty @@ -876,7 +876,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { filter: bson.D{{"_id", "array"}}, limit: 3, len: 1, - queryPushdown: allPushdown, + filterPushdown: allPushdown, unsafeLimitPushdown: false, }, "ValueFilter": { @@ -884,28 +884,28 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort: bson.D{{"_id", 1}}, limit: 3, len: 3, - queryPushdown: pgPushdown, + filterPushdown: pgPushdown, unsafeLimitPushdown: false, }, "DotNotationFilter": { filter: bson.D{{"v.foo", 42}}, limit: 3, len: 3, - queryPushdown: noPushdown, + filterPushdown: noPushdown, unsafeLimitPushdown: false, }, "ObjectFilter": { filter: bson.D{{"v", bson.D{{"foo", nil}}}}, limit: 3, len: 1, - queryPushdown: noPushdown, + filterPushdown: noPushdown, unsafeLimitPushdown: false, }, "Sort": { sort: bson.D{{"_id", 1}}, limit: 2, len: 2, - queryPushdown: noPushdown, + filterPushdown: noPushdown, unsafeLimitPushdown: true, }, "IDFilterSort": { @@ -913,7 +913,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort: bson.D{{"_id", 1}}, limit: 3, len: 1, - queryPushdown: allPushdown, + filterPushdown: allPushdown, unsafeLimitPushdown: false, }, "ValueFilterSort": { @@ -921,7 +921,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort: bson.D{{"_id", 1}}, limit: 3, len: 3, - queryPushdown: pgPushdown, + filterPushdown: pgPushdown, unsafeLimitPushdown: false, }, "DotNotationFilterSort": { @@ -929,7 +929,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort: bson.D{{"_id", 1}}, limit: 3, len: 3, - queryPushdown: noPushdown, + filterPushdown: noPushdown, unsafeLimitPushdown: false, }, "ObjectFilterSort": { @@ -937,7 +937,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort: bson.D{{"_id", 1}}, limit: 3, len: 1, - queryPushdown: noPushdown, + filterPushdown: noPushdown, unsafeLimitPushdown: false, }, "Skip": { @@ -1000,7 +1000,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } - resultPushdown := tc.queryPushdown + resultPushdown := tc.filterPushdown if setup.FilterPushdownDisabled() { resultPushdown = noPushdown @@ -1011,8 +1011,8 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { unsafeLimitPushdown, _ := doc.Get("limitPushdown") assert.Equal(t, tc.unsafeLimitPushdown, unsafeLimitPushdown, msg) - queryPushdown, _ := ConvertDocument(t, res).Get("pushdown") - assert.Equal(t, resultPushdown.FilterPushdownExpected(t), queryPushdown, msg) + filterPushdown, _ := ConvertDocument(t, res).Get("pushdown") + assert.Equal(t, resultPushdown.FilterPushdownExpected(t), filterPushdown, msg) }) t.Run("Find", func(t *testing.T) { diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 23b5de144a78..808caf2de6b7 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -255,7 +255,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams selectClause := prepareSelectClause(meta.TableName, meta.Capped(), false) - var queryPushdown bool + var filterPushdown bool var whereClause string var args []any @@ -265,7 +265,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams v, _ := params.Filter.Get("_id") switch v.(type) { case string, types.ObjectID: - queryPushdown = true + filterPushdown = true whereClause = fmt.Sprintf(` WHERE %s = ?`, metadata.IDColumn) args = []any{string(must.NotFail(sjson.MarshalSingleValue(v)))} } @@ -315,7 +315,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return &backends.ExplainResult{ QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), - FilterPushdown: queryPushdown, + FilterPushdown: filterPushdown, UnsafeSortPushdown: unsafeSortPushdown, UnsafeLimitPushdown: unsafeLimitPushdown, }, nil From cb7b85d7431c4c30030b2d4c6636af7c022bd01f Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 14:50:14 +0100 Subject: [PATCH 03/25] unsafe removal part uno --- cmd/ferretdb/main.go | 14 ++++++++------ integration/pushdown.go | 2 +- integration/query_test.go | 2 +- integration/setup/listener.go | 2 +- integration/setup/setup.go | 5 +++-- integration/setup/test_helpers.go | 11 ++++++++--- internal/backends/collection.go | 8 ++++---- internal/backends/collection_test.go | 12 ++++++------ internal/backends/postgresql/collection.go | 4 ++-- internal/backends/sqlite/collection.go | 14 +++++++------- internal/backends/sqlite/collection_test.go | 2 +- internal/handlers/registry/postgresql.go | 6 +++--- internal/handlers/registry/registry.go | 7 ++++--- internal/handlers/registry/sqlite.go | 6 +++--- internal/handlers/sqlite/msg_aggregate.go | 2 +- internal/handlers/sqlite/msg_explain.go | 12 ++++++------ internal/handlers/sqlite/msg_find.go | 6 +++--- internal/handlers/sqlite/sqlite.go | 7 ++++--- 18 files changed, 66 insertions(+), 56 deletions(-) diff --git a/cmd/ferretdb/main.go b/cmd/ferretdb/main.go index 80538d60cf7e..7d5a14e26b56 100644 --- a/cmd/ferretdb/main.go +++ b/cmd/ferretdb/main.go @@ -86,9 +86,10 @@ var cli struct { Test struct { RecordsDir string `default:"" help:"Testing: directory for record files."` - DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` - EnableUnsafeSortPushdown bool `default:"false" help:"Experimental: enable unsafe sort pushdown."` - EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` + DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` + EnableSortPushdown bool `default:"false" help:"Experimental: enable sort pushdown."` + EnableUnsafePushdown bool `default:"false" help:"Experimental: enable unsafe pushdown for both filter and sort."` + EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` //nolint:lll // for readability Telemetry struct { @@ -373,9 +374,10 @@ func run() { HANAURL: hanaFlags.HANAURL, TestOpts: registry.TestOpts{ - DisableFilterPushdown: cli.Test.DisableFilterPushdown, - EnableUnsafeSortPushdown: cli.Test.EnableUnsafeSortPushdown, - EnableOplog: cli.Test.EnableOplog, + DisableFilterPushdown: cli.Test.DisableFilterPushdown, + EnableSortPushdown: cli.Test.EnableSortPushdown, + EnableUnsafePushdown: cli.Test.EnableUnsafePushdown, + EnableOplog: cli.Test.EnableOplog, }, }) if err != nil { diff --git a/integration/pushdown.go b/integration/pushdown.go index 21a4cb2492a5..90edeb7e3df2 100644 --- a/integration/pushdown.go +++ b/integration/pushdown.go @@ -58,7 +58,7 @@ func (res resultPushdown) FilterPushdownExpected(t testtb.TB) bool { // It checks if pushdown is enabled by flag. // For capped collection, pushdown for recordID is done even if pushdown is not enabled by flag. func (res resultPushdown) SortPushdownExpected(t testtb.TB, cappedCollection bool) bool { - if !setup.UnsafeSortPushdownEnabled() && cappedCollection { + if !setup.SortPushdownEnabled() && cappedCollection { res = allPushdown } diff --git a/integration/query_test.go b/integration/query_test.go index 52ffb4872cab..d52d42564ed0 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -995,7 +995,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { var msg string - if !setup.UnsafeSortPushdownEnabled() && tc.sort != nil { + if !setup.SortPushdownEnabled() && tc.sort != nil { tc.unsafeLimitPushdown = false msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 605329ae5e91..0d962030cb89 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -227,7 +227,7 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string TestOpts: registry.TestOpts{ DisableFilterPushdown: *disableFilterPushdownF, - EnableUnsafeSortPushdown: *enableUnsafeSortPushdownF, + EnableUnsafeSortPushdown: *enableSortPushdownF, EnableOplog: true, }, } diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 2a340cd4b3e8..88c4ed79973b 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -59,8 +59,9 @@ var ( debugSetupF = flag.Bool("debug-setup", false, "enable debug logs for tests setup") logLevelF = zap.LevelFlag("log-level", zap.DebugLevel, "log level for tests") - disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") - enableUnsafeSortPushdownF = flag.Bool("enable-unsafe-sort-pushdown", false, "enable unsafe sort pushdown") + disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") + enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable unsafe sort pushdown") + enableUnsafePushdown = flag.Bool("enabe-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") ) // Other globals. diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index 484e4bdb51a2..545e8f9141d9 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -101,7 +101,12 @@ func FilterPushdownDisabled() bool { return *disableFilterPushdownF } -// UnsafeSortPushdownEnabled returns true if unsafe sort pushdown is enabled. -func UnsafeSortPushdownEnabled() bool { - return *enableUnsafeSortPushdownF +// SortPushdownEnabled returns true if sort pushdown is enabled. +func SortPushdownEnabled() bool { + return *enableSortPushdownF +} + +// UnsafePushdwon returns true if unsafe pushdown is enabled. +func UnsafePushdown() bool { + return *enableUnsafePushdown } diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 3a8ebbbb11c5..339d27f9d6fa 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -213,10 +213,10 @@ type ExplainParams struct { // ExplainResult represents the results of Collection.Explain method. type ExplainResult struct { - QueryPlanner *types.Document - FilterPushdown bool - UnsafeSortPushdown bool - UnsafeLimitPushdown bool + QueryPlanner *types.Document + FilterPushdown bool + SortPushdown bool + LimitPushdown bool } // Explain return a backend-specific execution plan for the given query. diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index 29bbc6ef930b..d46a0118fd53 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -94,7 +94,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, new(backends.ExplainParams)) require.NoError(t, err) - assert.True(t, explainRes.UnsafeSortPushdown) + assert.True(t, explainRes.SortPushdown) }) t.Run("CappedCollectionOnlyRecordIDs", func(t *testing.T) { @@ -110,7 +110,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, new(backends.ExplainParams)) require.NoError(t, err) - assert.True(t, explainRes.UnsafeSortPushdown) + assert.True(t, explainRes.SortPushdown) }) t.Run("CappedCollectionSortAsc", func(t *testing.T) { @@ -132,7 +132,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, &backends.ExplainParams{Sort: &sort}) require.NoError(t, err) - assert.True(t, explainRes.UnsafeSortPushdown) + assert.True(t, explainRes.SortPushdown) }) t.Run("CappedCollectionSortDesc", func(t *testing.T) { @@ -154,7 +154,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, &backends.ExplainParams{Sort: &sort}) require.NoError(t, err) - assert.True(t, explainRes.UnsafeSortPushdown) + assert.True(t, explainRes.SortPushdown) }) t.Run("CappedCollectionFilter", func(t *testing.T) { @@ -174,7 +174,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, &backends.ExplainParams{Filter: filter}) require.NoError(t, err) assert.True(t, explainRes.FilterPushdown) - assert.True(t, explainRes.UnsafeSortPushdown) + assert.True(t, explainRes.SortPushdown) }) t.Run("NonCappedCollectionOnlyRecordID", func(t *testing.T) { @@ -192,7 +192,7 @@ func TestCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := coll.Explain(ctx, new(backends.ExplainParams)) require.NoError(t, err) - assert.False(t, explainRes.UnsafeSortPushdown) + assert.False(t, explainRes.SortPushdown) }) }) } diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index b3ddb808747c..89f3e45114ea 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -313,12 +313,12 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams sort, sortArgs := prepareOrderByClause(&placeholder, params.Sort, meta.Capped()) q += sort args = append(args, sortArgs...) - res.UnsafeSortPushdown = sort != "" + res.SortPushdown = sort != "" if params.Limit != 0 { q += fmt.Sprintf(` LIMIT %s`, placeholder.Next()) args = append(args, params.Limit) - res.UnsafeLimitPushdown = true + res.LimitPushdown = true } var b []byte diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 808caf2de6b7..ce7fd4ca3d9a 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -272,16 +272,16 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams } orderByClause := prepareOrderByClause(params.Sort, meta.Capped()) - unsafeSortPushdown := orderByClause != "" + sortPushdown := orderByClause != "" q := `EXPLAIN QUERY PLAN ` + selectClause + whereClause + orderByClause - var unsafeLimitPushdown bool + var limitPushdown bool if params.Limit != 0 { q += ` LIMIT ?` args = append(args, params.Limit) - unsafeLimitPushdown = true + limitPushdown = true } rows, err := db.QueryContext(ctx, q, args...) @@ -314,10 +314,10 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams } return &backends.ExplainResult{ - QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), - FilterPushdown: filterPushdown, - UnsafeSortPushdown: unsafeSortPushdown, - UnsafeLimitPushdown: unsafeLimitPushdown, + QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), + FilterPushdown: filterPushdown, + SortPushdown: sortPushdown, + LimitPushdown: limitPushdown, }, nil } diff --git a/internal/backends/sqlite/collection_test.go b/internal/backends/sqlite/collection_test.go index 901ea07ffe27..ae9b0cddad8b 100644 --- a/internal/backends/sqlite/collection_test.go +++ b/internal/backends/sqlite/collection_test.go @@ -82,6 +82,6 @@ func TestCappedCollectionInsertAllQueryExplain(t *testing.T) { explainRes, err := cappedColl.Explain(ctx, &backends.ExplainParams{Sort: &sort}) require.NoError(t, err) - assert.False(t, explainRes.UnsafeSortPushdown) + assert.False(t, explainRes.SortPushdown) }) } diff --git a/internal/handlers/registry/postgresql.go b/internal/handlers/registry/postgresql.go index 342636d8bc16..5d31afbf577b 100644 --- a/internal/handlers/registry/postgresql.go +++ b/internal/handlers/registry/postgresql.go @@ -30,9 +30,9 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableUnsafeSortPushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index 661cd7a62a53..0272030c83e8 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -55,9 +55,10 @@ type NewHandlerOpts struct { // TestOpts represents experimental configuration options. type TestOpts struct { - DisableFilterPushdown bool - EnableUnsafeSortPushdown bool - EnableOplog bool + DisableFilterPushdown bool + EnableSortPushdown bool + EnableUnsafePushdown bool + EnableOplog bool } // NewHandler constructs a new handler. diff --git a/internal/handlers/registry/sqlite.go b/internal/handlers/registry/sqlite.go index 19519e525c26..25902329afd9 100644 --- a/internal/handlers/registry/sqlite.go +++ b/internal/handlers/registry/sqlite.go @@ -30,9 +30,9 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableUnsafeSortPushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 352f1dfc1940..341082d55286 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -269,7 +269,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs } // Skip sorting if there are more than one sort parameters - if h.EnableUnsafeSortPushdown && sort.Len() == 1 { + if h.EnableSortPushdown && sort.Len() == 1 { var order types.SortType k := sort.Keys()[0] diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 7c15810ca5b3..98c1852b29ce 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -88,7 +88,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } // Skip sorting if there are more than one sort parameters - if h.EnableUnsafeSortPushdown && params.Sort.Len() == 1 { + if h.EnableSortPushdown && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -107,10 +107,10 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `UnsafeSortPushdown` is not set, it must fetch all documents + // - `sort` is set but `EnableSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } @@ -118,7 +118,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, qp.Filter = nil } - if !h.EnableUnsafeSortPushdown { + if !h.EnableSortPushdown { qp.Sort = nil } @@ -138,8 +138,8 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // our extensions // TODO https://github.com/FerretDB/FerretDB/issues/3235 "pushdown", res.FilterPushdown, - "sortingPushdown", res.UnsafeSortPushdown, - "limitPushdown", res.UnsafeLimitPushdown, + "sortingPushdown", res.SortPushdown, + "limitPushdown", res.LimitPushdown, "ok", float64(1), ))}, diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index 2f5ab567ceec..26ee89a50b16 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -81,7 +81,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er } // Skip sorting if there are more than one sort parameters - if h.EnableUnsafeSortPushdown && params.Sort.Len() == 1 { + if h.EnableSortPushdown && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -100,10 +100,10 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `UnsafeSortPushdown` is not set, it must fetch all documents + // - `sort` is set but `EnableSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index eba126e8b2e1..383eeb62b266 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -53,9 +53,10 @@ type NewOpts struct { StateProvider *state.Provider // test options - DisableFilterPushdown bool - EnableUnsafeSortPushdown bool - EnableOplog bool + DisableFilterPushdown bool + EnableSortPushdown bool + EnableUnsafePushdown bool + EnableOplog bool } // New returns a new handler. From 143c9dd60e33945fac2fbbd6ff9d0b10939aa472 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 15:45:48 +0100 Subject: [PATCH 04/25] unsafe removal part dos --- Taskfile.yml | 12 +- integration/query_compat_test.go | 24 ++-- integration/query_test.go | 166 +++++++++++------------ integration/setup/listener.go | 7 +- internal/handlers/registry/hana.go | 7 +- internal/handlers/registry/postgresql.go | 3 +- internal/handlers/registry/sqlite.go | 3 +- 7 files changed, 115 insertions(+), 107 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3a3bc5c6b0f7..0d367bfc753a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,7 +9,8 @@ vars: SHARD_INDEX: 1 SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false - ENABLE_UNSAFE_SORT_PUSHDOWN: false + ENABLE_SORT_PUSHDOWN: false + ENABLE_UNSAFE_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m BENCH_TIME: 5s @@ -218,7 +219,8 @@ tasks: -postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?search_path=' -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} + -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-sqlite: desc: "Run integration tests for `sqlite` backend" @@ -244,7 +246,8 @@ tasks: -target-tls -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} + -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-hana: desc: "Run integration tests for `hana` handler" @@ -270,7 +273,8 @@ tasks: -hana-url=$FERRETDB_HANA_URL -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} + -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-mongodb: desc: "Run integration tests for MongoDB" diff --git a/integration/query_compat_test.go b/integration/query_compat_test.go index 6eb843a0c717..5f59dd3ee327 100644 --- a/integration/query_compat_test.go +++ b/integration/query_compat_test.go @@ -242,29 +242,29 @@ func TestQueryCappedCollectionCompat(t *testing.T) { filter bson.D sort bson.D - unsafeSortPushdown resultPushdown + sortPushdown resultPushdown }{ "NoSortNoFilter": { - unsafeSortPushdown: allPushdown, + sortPushdown: allPushdown, }, "Filter": { - filter: bson.D{{"v", int32(42)}}, - unsafeSortPushdown: allPushdown, + filter: bson.D{{"v", int32(42)}}, + sortPushdown: allPushdown, }, "Sort": { - sort: bson.D{{"_id", int32(-1)}}, - unsafeSortPushdown: pgPushdown, + sort: bson.D{{"_id", int32(-1)}}, + sortPushdown: pgPushdown, }, "FilterSort": { - filter: bson.D{{"v", int32(42)}}, - sort: bson.D{{"_id", int32(-1)}}, - unsafeSortPushdown: pgPushdown, + filter: bson.D{{"v", int32(42)}}, + sort: bson.D{{"_id", int32(-1)}}, + sortPushdown: pgPushdown, }, "MultipleSortFields": { sort: bson.D{{"v", 1}, {"_id", int32(-1)}}, // multiple sort fields are skipped by handler and no sort pushdown // is set on handler, so record ID pushdown is done. - unsafeSortPushdown: allPushdown, + sortPushdown: allPushdown, }, } { name, tc := name, tc @@ -288,8 +288,8 @@ func TestQueryCappedCollectionCompat(t *testing.T) { require.NoError(t, targetCollection.Database().RunCommand(ctx, bson.D{{"explain", explainQuery}}).Decode(&explainRes)) doc := ConvertDocument(t, explainRes) - unsafeSortPushdown, _ := doc.Get("sortingPushdown") - assert.Equal(t, tc.unsafeSortPushdown.SortPushdownExpected(t, true), unsafeSortPushdown) + sortPushdown, _ := doc.Get("sortingPushdown") + assert.Equal(t, tc.sortPushdown.SortPushdownExpected(t, true), sortPushdown) findOpts := options.Find() if tc.sort != nil { diff --git a/integration/query_test.go b/integration/query_test.go index d52d42564ed0..5ebcdbe4e735 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -821,7 +821,7 @@ func TestQueryCommandSingleBatch(t *testing.T) { } } -func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { +func TestQueryCommandLimitPushDown(t *testing.T) { t.Parallel() // must use a collection of documents which does not support filter pushdown to test limit pushdown @@ -834,117 +834,117 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { sort bson.D // optional, nil to leave sort unset optSkip *int64 // optional, nil to leave optSkip unset - len int // expected length of results - filterPushdown resultPushdown // optional, defaults to noPushdown - unsafeLimitPushdown bool // optional, set true for expected pushdown for limit - err *mongo.CommandError // optional, expected error from MongoDB - altMessage string // optional, alternative error message for FerretDB, ignored if empty - skip string // optional, skip test with a specified reason - failsForSQLite string // optional, if set, the case is expected to fail for SQLite due to given issue + len int // expected length of results + filterPushdown resultPushdown // optional, defaults to noPushdown + limitPushdown bool // optional, set true for expected pushdown for limit + err *mongo.CommandError // optional, expected error from MongoDB + altMessage string // optional, alternative error message for FerretDB, ignored if empty + skip string // optional, skip test with a specified reason + failsForSQLite string // optional, if set, the case is expected to fail for SQLite due to given issue }{ "Simple": { - limit: 1, - len: 1, - unsafeLimitPushdown: true, + limit: 1, + len: 1, + limitPushdown: true, }, "AlmostAll": { - limit: int64(len(shareddata.Composites.Docs()) - 1), - len: len(shareddata.Composites.Docs()) - 1, - unsafeLimitPushdown: true, + limit: int64(len(shareddata.Composites.Docs()) - 1), + len: len(shareddata.Composites.Docs()) - 1, + limitPushdown: true, }, "All": { - limit: int64(len(shareddata.Composites.Docs())), - len: len(shareddata.Composites.Docs()), - unsafeLimitPushdown: true, + limit: int64(len(shareddata.Composites.Docs())), + len: len(shareddata.Composites.Docs()), + limitPushdown: true, }, "More": { - limit: int64(len(shareddata.Composites.Docs()) + 1), - len: len(shareddata.Composites.Docs()), - unsafeLimitPushdown: true, + limit: int64(len(shareddata.Composites.Docs()) + 1), + len: len(shareddata.Composites.Docs()), + limitPushdown: true, }, "Big": { - limit: 1000, - len: len(shareddata.Composites.Docs()), - unsafeLimitPushdown: true, + limit: 1000, + len: len(shareddata.Composites.Docs()), + limitPushdown: true, }, "Zero": { - limit: 0, - len: len(shareddata.Composites.Docs()), - unsafeLimitPushdown: false, + limit: 0, + len: len(shareddata.Composites.Docs()), + limitPushdown: false, }, "IDFilter": { - filter: bson.D{{"_id", "array"}}, - limit: 3, - len: 1, - filterPushdown: allPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"_id", "array"}}, + limit: 3, + len: 1, + filterPushdown: allPushdown, + limitPushdown: false, }, "ValueFilter": { - filter: bson.D{{"v", 42}}, - sort: bson.D{{"_id", 1}}, - limit: 3, - len: 3, - filterPushdown: pgPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v", 42}}, + sort: bson.D{{"_id", 1}}, + limit: 3, + len: 3, + filterPushdown: pgPushdown, + limitPushdown: false, }, "DotNotationFilter": { - filter: bson.D{{"v.foo", 42}}, - limit: 3, - len: 3, - filterPushdown: noPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v.foo", 42}}, + limit: 3, + len: 3, + filterPushdown: noPushdown, + limitPushdown: false, }, "ObjectFilter": { - filter: bson.D{{"v", bson.D{{"foo", nil}}}}, - limit: 3, - len: 1, - filterPushdown: noPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v", bson.D{{"foo", nil}}}}, + limit: 3, + len: 1, + filterPushdown: noPushdown, + limitPushdown: false, }, "Sort": { - sort: bson.D{{"_id", 1}}, - limit: 2, - len: 2, - filterPushdown: noPushdown, - unsafeLimitPushdown: true, + sort: bson.D{{"_id", 1}}, + limit: 2, + len: 2, + filterPushdown: noPushdown, + limitPushdown: true, }, "IDFilterSort": { - filter: bson.D{{"_id", "array"}}, - sort: bson.D{{"_id", 1}}, - limit: 3, - len: 1, - filterPushdown: allPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"_id", "array"}}, + sort: bson.D{{"_id", 1}}, + limit: 3, + len: 1, + filterPushdown: allPushdown, + limitPushdown: false, }, "ValueFilterSort": { - filter: bson.D{{"v", 42}}, - sort: bson.D{{"_id", 1}}, - limit: 3, - len: 3, - filterPushdown: pgPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v", 42}}, + sort: bson.D{{"_id", 1}}, + limit: 3, + len: 3, + filterPushdown: pgPushdown, + limitPushdown: false, }, "DotNotationFilterSort": { - filter: bson.D{{"v.foo", 42}}, - sort: bson.D{{"_id", 1}}, - limit: 3, - len: 3, - filterPushdown: noPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v.foo", 42}}, + sort: bson.D{{"_id", 1}}, + limit: 3, + len: 3, + filterPushdown: noPushdown, + limitPushdown: false, }, "ObjectFilterSort": { - filter: bson.D{{"v", bson.D{{"foo", nil}}}}, - sort: bson.D{{"_id", 1}}, - limit: 3, - len: 1, - filterPushdown: noPushdown, - unsafeLimitPushdown: false, + filter: bson.D{{"v", bson.D{{"foo", nil}}}}, + sort: bson.D{{"_id", 1}}, + limit: 3, + len: 1, + filterPushdown: noPushdown, + limitPushdown: false, }, "Skip": { - optSkip: pointer.ToInt64(1), - limit: 2, - len: 2, - unsafeLimitPushdown: false, + optSkip: pointer.ToInt64(1), + limit: 2, + len: 2, + limitPushdown: false, }, } { tc, name := tc, name @@ -996,7 +996,7 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { var msg string if !setup.SortPushdownEnabled() && tc.sort != nil { - tc.unsafeLimitPushdown = false + tc.limitPushdown = false msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } @@ -1008,8 +1008,8 @@ func TestQueryCommandUnsafeLimitPushDown(t *testing.T) { } doc := ConvertDocument(t, res) - unsafeLimitPushdown, _ := doc.Get("limitPushdown") - assert.Equal(t, tc.unsafeLimitPushdown, unsafeLimitPushdown, msg) + limitPushdown, _ := doc.Get("limitPushdown") + assert.Equal(t, tc.limitPushdown, limitPushdown, msg) filterPushdown, _ := ConvertDocument(t, res).Get("pushdown") assert.Equal(t, resultPushdown.FilterPushdownExpected(t), filterPushdown, msg) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index 0d962030cb89..e192cb79da38 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -226,9 +226,10 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string HANAURL: *hanaURLF, TestOpts: registry.TestOpts{ - DisableFilterPushdown: *disableFilterPushdownF, - EnableUnsafeSortPushdown: *enableSortPushdownF, - EnableOplog: true, + DisableFilterPushdown: *disableFilterPushdownF, + EnableSortPushdown: *enableSortPushdownF, + EnableUnsafePushdown: *enableUnsafePushdownF, + EnableOplog: true, }, } h, err := registry.NewHandler(handler, handlerOpts) diff --git a/internal/handlers/registry/hana.go b/internal/handlers/registry/hana.go index 59fe68cc6b9f..ef7bb0a79638 100644 --- a/internal/handlers/registry/hana.go +++ b/internal/handlers/registry/hana.go @@ -34,9 +34,10 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafePushdown: opts.EnableUnsafePushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/registry/postgresql.go b/internal/handlers/registry/postgresql.go index 5d31afbf577b..048661146b14 100644 --- a/internal/handlers/registry/postgresql.go +++ b/internal/handlers/registry/postgresql.go @@ -31,7 +31,8 @@ func init() { StateProvider: opts.StateProvider, DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableUnsafeSortPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafePushdown: opts.EnableUnsafePushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/registry/sqlite.go b/internal/handlers/registry/sqlite.go index 25902329afd9..d02e9fd903a5 100644 --- a/internal/handlers/registry/sqlite.go +++ b/internal/handlers/registry/sqlite.go @@ -31,7 +31,8 @@ func init() { StateProvider: opts.StateProvider, DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableUnsafeSortPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafePushdown: opts.EnableUnsafePushdown, EnableOplog: opts.EnableOplog, } From 6926153ae5ce2a2353c5f3d780a2e7303e7584a7 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 15:51:10 +0100 Subject: [PATCH 05/25] unsafe removal part drei --- integration/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 88c4ed79973b..c814f301d863 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -61,7 +61,7 @@ var ( disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable unsafe sort pushdown") - enableUnsafePushdown = flag.Bool("enabe-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") + enableUnsafePushdownF = flag.Bool("enabe-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") ) // Other globals. From a68f0563cec65d143adb7502b62771c43bed67f1 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 9 Nov 2023 16:31:38 +0100 Subject: [PATCH 06/25] unsafe removal part quatro --- integration/setup/test_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index 545e8f9141d9..4beb7d032373 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -108,5 +108,5 @@ func SortPushdownEnabled() bool { // UnsafePushdwon returns true if unsafe pushdown is enabled. func UnsafePushdown() bool { - return *enableUnsafePushdown + return *enableUnsafePushdownF } From 24dc52a9c8a613681cb69aee10d8b19110f3a85d Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 14 Nov 2023 23:28:11 +0100 Subject: [PATCH 07/25] wip --- integration/setup/setup.go | 2 +- integration/setup/test_helpers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 6ff05665af25..1e5d0e04b03b 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -61,7 +61,7 @@ var ( disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable unsafe sort pushdown") - enableUnsafePushdownF = flag.Bool("enabe-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") + enableUnsafePushdownF = flag.Bool("enable-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") ) // Other globals. diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index 4beb7d032373..c0a5e66f82b4 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -106,7 +106,7 @@ func SortPushdownEnabled() bool { return *enableSortPushdownF } -// UnsafePushdwon returns true if unsafe pushdown is enabled. +// UnsafePushdown returns true if unsafe pushdown is enabled. func UnsafePushdown() bool { return *enableUnsafePushdownF } From 15a207722fb6bc7b5f97b2d7a06992a2e368f625 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 10:07:18 +0100 Subject: [PATCH 08/25] wip --- Taskfile.yml | 4 ---- cmd/ferretdb/main.go | 2 -- integration/setup/listener.go | 1 - integration/setup/setup.go | 3 +-- integration/setup/test_helpers.go | 5 ----- internal/handlers/registry/hana.go | 1 - internal/handlers/registry/postgresql.go | 1 - internal/handlers/registry/registry.go | 1 - internal/handlers/registry/sqlite.go | 1 - internal/handlers/sqlite/sqlite.go | 1 - 10 files changed, 1 insertion(+), 19 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 344c7b831511..d69b3f01080f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,7 +10,6 @@ vars: SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false ENABLE_SORT_PUSHDOWN: false - ENABLE_UNSAFE_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m BENCH_TIME: 5s @@ -220,7 +219,6 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} - -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-sqlite: desc: "Run integration tests for `sqlite` backend" @@ -247,7 +245,6 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} - -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-hana: desc: "Run integration tests for `hana` handler" @@ -274,7 +271,6 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} - -enable-unsafe-pushdown={{.ENABLE_UNSAFE_PUSHDOWN}} test-integration-mongodb: desc: "Run integration tests for MongoDB" diff --git a/cmd/ferretdb/main.go b/cmd/ferretdb/main.go index 21df74a3aa94..0bf939368bf4 100644 --- a/cmd/ferretdb/main.go +++ b/cmd/ferretdb/main.go @@ -88,7 +88,6 @@ var cli struct { DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` EnableSortPushdown bool `default:"false" help:"Experimental: enable sort pushdown."` - EnableUnsafePushdown bool `default:"false" help:"Experimental: enable unsafe pushdown for both filter and sort."` EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` //nolint:lll // for readability @@ -384,7 +383,6 @@ func run() { TestOpts: registry.TestOpts{ DisableFilterPushdown: cli.Test.DisableFilterPushdown, EnableSortPushdown: cli.Test.EnableSortPushdown, - EnableUnsafePushdown: cli.Test.EnableUnsafePushdown, EnableOplog: cli.Test.EnableOplog, }, }) diff --git a/integration/setup/listener.go b/integration/setup/listener.go index c2564d066759..a61a8e76b6cb 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -161,7 +161,6 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string TestOpts: registry.TestOpts{ DisableFilterPushdown: *disableFilterPushdownF, EnableSortPushdown: *enableSortPushdownF, - EnableUnsafePushdown: *enableUnsafePushdownF, EnableOplog: true, }, } diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 1e5d0e04b03b..379719f72685 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -60,8 +60,7 @@ var ( logLevelF = zap.LevelFlag("log-level", zap.DebugLevel, "log level for tests") disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") - enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable unsafe sort pushdown") - enableUnsafePushdownF = flag.Bool("enable-unsafe-pushdown", false, "enable unsafe pushdown for both filter and sort") + enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable sort pushdown") ) // Other globals. diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index c0a5e66f82b4..0c03bdaf2c86 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -105,8 +105,3 @@ func FilterPushdownDisabled() bool { func SortPushdownEnabled() bool { return *enableSortPushdownF } - -// UnsafePushdown returns true if unsafe pushdown is enabled. -func UnsafePushdown() bool { - return *enableUnsafePushdownF -} diff --git a/internal/handlers/registry/hana.go b/internal/handlers/registry/hana.go index ef7bb0a79638..f8845b1c53e3 100644 --- a/internal/handlers/registry/hana.go +++ b/internal/handlers/registry/hana.go @@ -36,7 +36,6 @@ func init() { DisableFilterPushdown: opts.DisableFilterPushdown, EnableSortPushdown: opts.EnableSortPushdown, - EnableUnsafePushdown: opts.EnableUnsafePushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/registry/postgresql.go b/internal/handlers/registry/postgresql.go index 048661146b14..1bd079fb6fe0 100644 --- a/internal/handlers/registry/postgresql.go +++ b/internal/handlers/registry/postgresql.go @@ -32,7 +32,6 @@ func init() { DisableFilterPushdown: opts.DisableFilterPushdown, EnableSortPushdown: opts.EnableSortPushdown, - EnableUnsafePushdown: opts.EnableUnsafePushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index 0272030c83e8..b90195236641 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -57,7 +57,6 @@ type NewHandlerOpts struct { type TestOpts struct { DisableFilterPushdown bool EnableSortPushdown bool - EnableUnsafePushdown bool EnableOplog bool } diff --git a/internal/handlers/registry/sqlite.go b/internal/handlers/registry/sqlite.go index d02e9fd903a5..127a02d1f78c 100644 --- a/internal/handlers/registry/sqlite.go +++ b/internal/handlers/registry/sqlite.go @@ -32,7 +32,6 @@ func init() { DisableFilterPushdown: opts.DisableFilterPushdown, EnableSortPushdown: opts.EnableSortPushdown, - EnableUnsafePushdown: opts.EnableUnsafePushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 383eeb62b266..7323b4473c36 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -55,7 +55,6 @@ type NewOpts struct { // test options DisableFilterPushdown bool EnableSortPushdown bool - EnableUnsafePushdown bool EnableOplog bool } From 84ad078b5d4f56a9b03acdc6d12d7169a015a78e Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 16:34:57 +0100 Subject: [PATCH 09/25] separate flags --- Taskfile.yml | 4 ++++ cmd/ferretdb/main.go | 14 ++++++++------ integration/setup/listener.go | 11 ++++++++--- integration/setup/setup.go | 5 +++-- internal/handlers/registry/hana.go | 7 ++++--- internal/handlers/registry/postgresql.go | 7 ++++--- internal/handlers/registry/registry.go | 7 ++++--- internal/handlers/registry/sqlite.go | 7 ++++--- internal/handlers/sqlite/msg_aggregate.go | 2 +- internal/handlers/sqlite/msg_explain.go | 6 +++--- internal/handlers/sqlite/msg_find.go | 6 +++--- internal/handlers/sqlite/sqlite.go | 7 ++++--- 12 files changed, 50 insertions(+), 33 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index d69b3f01080f..b5b97bd98356 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,6 +10,7 @@ vars: SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false ENABLE_SORT_PUSHDOWN: false + ENABLE_UNSAFE_SORT_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m BENCH_TIME: 5s @@ -219,6 +220,7 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-sqlite: desc: "Run integration tests for `sqlite` backend" @@ -245,6 +247,7 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-hana: desc: "Run integration tests for `hana` handler" @@ -271,6 +274,7 @@ tasks: -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} + -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-mongodb: desc: "Run integration tests for MongoDB" diff --git a/cmd/ferretdb/main.go b/cmd/ferretdb/main.go index 0bf939368bf4..cbfcbc1f958e 100644 --- a/cmd/ferretdb/main.go +++ b/cmd/ferretdb/main.go @@ -86,9 +86,10 @@ var cli struct { Test struct { RecordsDir string `default:"" help:"Testing: directory for record files."` - DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` - EnableSortPushdown bool `default:"false" help:"Experimental: enable sort pushdown."` - EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` + DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` + EnableSortPushdown bool `default:"false" help:"Experimental: enable sort pushdown."` + EnableUnsafeSortPushdown bool `default:"false" help:"Experimental: enable unsafe sort pushdown."` + EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` //nolint:lll // for readability Telemetry struct { @@ -381,9 +382,10 @@ func run() { HANAURL: hanaFlags.HANAURL, TestOpts: registry.TestOpts{ - DisableFilterPushdown: cli.Test.DisableFilterPushdown, - EnableSortPushdown: cli.Test.EnableSortPushdown, - EnableOplog: cli.Test.EnableOplog, + DisableFilterPushdown: cli.Test.DisableFilterPushdown, + EnableSortPushdown: cli.Test.EnableSortPushdown, + EnableUnsafeSortPushdown: cli.Test.EnableUnsafeSortPushdown, + EnableOplog: cli.Test.EnableOplog, }, }) if err != nil { diff --git a/integration/setup/listener.go b/integration/setup/listener.go index a61a8e76b6cb..3e6a003edb30 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -159,9 +159,10 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string HANAURL: *hanaURLF, TestOpts: registry.TestOpts{ - DisableFilterPushdown: *disableFilterPushdownF, - EnableSortPushdown: *enableSortPushdownF, - EnableOplog: true, + DisableFilterPushdown: *disableFilterPushdownF, + EnableSortPushdown: *enableSortPushdownF, + EnableUnsafeSortPushdown: *enableUnsafeSortPushdownF, + EnableOplog: true, }, } h, err := registry.NewHandler(handler, handlerOpts) @@ -184,6 +185,10 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string tb.Fatal("Both -target-tls and -target-unix-socket are set.") } + if *enableSortPushdownF && *enableUnsafeSortPushdownF { + tb.Fatal("Both -enable-sort-pushdown and -enable-unsafe-sort-pushdown are set.") + } + switch { case *targetTLSF: listenerOpts.TLS = "127.0.0.1:0" diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 379719f72685..98d0d2737989 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -59,8 +59,9 @@ var ( debugSetupF = flag.Bool("debug-setup", false, "enable debug logs for tests setup") logLevelF = zap.LevelFlag("log-level", zap.DebugLevel, "log level for tests") - disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") - enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable sort pushdown") + disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") + enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable sort pushdown") + enableUnsafeSortPushdownF = flag.Bool("enable-unsafe-sort-pushdown", false, "enable unsafe sort pushdown") ) // Other globals. diff --git a/internal/handlers/registry/hana.go b/internal/handlers/registry/hana.go index f8845b1c53e3..b905832ed717 100644 --- a/internal/handlers/registry/hana.go +++ b/internal/handlers/registry/hana.go @@ -34,9 +34,10 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/registry/postgresql.go b/internal/handlers/registry/postgresql.go index 1bd079fb6fe0..0a484282f41d 100644 --- a/internal/handlers/registry/postgresql.go +++ b/internal/handlers/registry/postgresql.go @@ -30,9 +30,10 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index b90195236641..a6c8fbb1c199 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -55,9 +55,10 @@ type NewHandlerOpts struct { // TestOpts represents experimental configuration options. type TestOpts struct { - DisableFilterPushdown bool - EnableSortPushdown bool - EnableOplog bool + DisableFilterPushdown bool + EnableSortPushdown bool + EnableUnsafeSortPushdown bool + EnableOplog bool } // NewHandler constructs a new handler. diff --git a/internal/handlers/registry/sqlite.go b/internal/handlers/registry/sqlite.go index 127a02d1f78c..661d59a6bcf7 100644 --- a/internal/handlers/registry/sqlite.go +++ b/internal/handlers/registry/sqlite.go @@ -30,9 +30,10 @@ func init() { ConnMetrics: opts.ConnMetrics, StateProvider: opts.StateProvider, - DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, - EnableOplog: opts.EnableOplog, + DisableFilterPushdown: opts.DisableFilterPushdown, + EnableSortPushdown: opts.EnableSortPushdown, + EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, + EnableOplog: opts.EnableOplog, } return sqlite.New(handlerOpts) diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 0b6096b5e402..3b28d385fa25 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -269,7 +269,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs } // Skip sorting if there are more than one sort parameters - if h.EnableSortPushdown && sort.Len() == 1 { + if h.EnableUnsafeSortPushdown && sort.Len() == 1 { var order types.SortType k := sort.Keys()[0] diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 98c1852b29ce..a4cb0f0f04e3 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -88,7 +88,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } // Skip sorting if there are more than one sort parameters - if h.EnableSortPushdown && params.Sort.Len() == 1 { + if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -107,10 +107,10 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `EnableSortPushdown` is not set, it must fetch all documents + // - `sort` is set but `EnableUnsafeSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index 868d02364d89..cf393e049f88 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -113,7 +113,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er } // Skip sorting if there are more than one sort parameters - if h.EnableSortPushdown && params.Sort.Len() == 1 { + if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -132,10 +132,10 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `EnableSortPushdown` is not set, it must fetch all documents + // - `sort` is set but `EnableUnsafeSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 7323b4473c36..7bcc4cbc8268 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -53,9 +53,10 @@ type NewOpts struct { StateProvider *state.Provider // test options - DisableFilterPushdown bool - EnableSortPushdown bool - EnableOplog bool + DisableFilterPushdown bool + EnableSortPushdown bool + EnableUnsafeSortPushdown bool + EnableOplog bool } // New returns a new handler. From 94ee9842a2b4c43bca9bca2f6cfb85d6afb3e418 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 17:37:18 +0100 Subject: [PATCH 10/25] wip --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index b5b97bd98356..a794a8e3982e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,7 +9,7 @@ vars: SHARD_INDEX: 1 SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false - ENABLE_SORT_PUSHDOWN: false + ENABLE_SORT_PUSHDOWN: true ENABLE_UNSAFE_SORT_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m From 9fce730e8f662c0e65b5b98440ecee0087e5d008 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 18:59:55 +0100 Subject: [PATCH 11/25] wip --- internal/handlers/sqlite/msg_explain.go | 4 ++-- internal/handlers/sqlite/msg_find.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index a4cb0f0f04e3..69df14716bab 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -107,10 +107,10 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `EnableUnsafeSortPushdown` is not set, it must fetch all documents + // - `sort` is set but sort pushdown is not enabled, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index cf393e049f88..d916cf32633f 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -132,10 +132,10 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but `EnableUnsafeSortPushdown` is not set, it must fetch all documents + // - `sort` is set but sort pushdown is not enabled, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } From bd2dcc280dbc2ae6663281556e1aaf80308ab2a4 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 19:07:19 +0100 Subject: [PATCH 12/25] wip --- internal/handlers/sqlite/msg_aggregate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 3b28d385fa25..ac770e4ce503 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -269,7 +269,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs } // Skip sorting if there are more than one sort parameters - if h.EnableUnsafeSortPushdown && sort.Len() == 1 { + if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && sort.Len() == 1 { var order types.SortType k := sort.Keys()[0] From 7e61b958a10da6b3b344b064c529183d0fba7e2c Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 19:14:06 +0100 Subject: [PATCH 13/25] wip --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index a794a8e3982e..b5b97bd98356 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,7 +9,7 @@ vars: SHARD_INDEX: 1 SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false - ENABLE_SORT_PUSHDOWN: true + ENABLE_SORT_PUSHDOWN: false ENABLE_UNSAFE_SORT_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m From 4a03d6181f34d3aa780b1839c8ee94d0630a9b89 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 19:39:34 +0100 Subject: [PATCH 14/25] wip --- .../aggregate_documents_compat_test.go | 42 ++++++++++--------- internal/handlers/sqlite/msg_aggregate.go | 1 + 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/integration/aggregate_documents_compat_test.go b/integration/aggregate_documents_compat_test.go index 4d813c04f3de..76597881c735 100644 --- a/integration/aggregate_documents_compat_test.go +++ b/integration/aggregate_documents_compat_test.go @@ -104,26 +104,6 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P t.Run(targetCollection.Name(), func(t *testing.T) { t.Helper() - explainCommand := bson.D{{"explain", bson.D{ - {"aggregate", targetCollection.Name()}, - {"pipeline", pipeline}, - }}} - var explainRes bson.D - require.NoError(t, targetCollection.Database().RunCommand(ctx, explainCommand).Decode(&explainRes)) - - resultPushdown := tc.resultPushdown - - var msg string - // TODO https://github.com/FerretDB/FerretDB/issues/3386 - if setup.FilterPushdownDisabled() { - resultPushdown = noPushdown - msg = "Fitler pushdown is disabled, but target resulted with pushdown" - } - - doc := ConvertDocument(t, explainRes) - pushdown, _ := doc.Get("pushdown") - assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) - targetCursor, targetErr := targetCollection.Aggregate(ctx, pipeline, opts) compatCursor, compatErr := compatCollection.Aggregate(ctx, pipeline, opts) @@ -153,6 +133,28 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P if len(targetRes) > 0 || len(compatRes) > 0 { nonEmptyResults = true } + + if targetErr == nil { + explainCommand := bson.D{{"explain", bson.D{ + {"aggregate", targetCollection.Name()}, + {"pipeline", pipeline}, + }}} + var explainRes bson.D + require.NoError(t, targetCollection.Database().RunCommand(ctx, explainCommand).Decode(&explainRes)) + + resultPushdown := tc.resultPushdown + + var msg string + // TODO https://github.com/FerretDB/FerretDB/issues/3386 + if setup.FilterPushdownDisabled() { + resultPushdown = noPushdown + msg = "Fitler pushdown is disabled, but target resulted with pushdown" + } + + doc := ConvertDocument(t, explainRes) + pushdown, _ := doc.Get("pushdown") + assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) + } }) } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index ac770e4ce503..5d841e5a4a52 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -277,6 +277,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs order, err = common.GetSortType(k, v) if err != nil { + closer.Close() return nil, err } From 069ea9af090ca49069b94485c40da6333e796bbf Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 20:13:37 +0100 Subject: [PATCH 15/25] wip --- internal/handlers/sqlite/msg_explain.go | 4 +++- internal/handlers/sqlite/msg_find.go | 4 +++- website/docs/pushdown.md | 12 ++++++++++++ website/docs/reference/glossary.md | 6 ++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 69df14716bab..2944920cbaff 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -110,7 +110,9 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // - `sort` is set but sort pushdown is not enabled, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && + (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && + params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index d916cf32633f..e8abb9e734cd 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -135,7 +135,9 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // - `sort` is set but sort pushdown is not enabled, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Skip == 0 { + if params.Filter.Len() == 0 && + (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && + params.Skip == 0 { qp.Limit = params.Limit } diff --git a/website/docs/pushdown.md b/website/docs/pushdown.md index c6f0f228cfcf..281d94a37616 100644 --- a/website/docs/pushdown.md +++ b/website/docs/pushdown.md @@ -17,6 +17,18 @@ To make this process more efficient, we minimize the amount of incoming data, by You can learn more about query pushdown in our [blog post](https://blog.ferretdb.io/ferretdb-fetches-data-query-pushdown/). ::: +## Types of pushdown + +Currently we support 3 different types of pushdown. + +The **Filter pushdown**, which is enabled by default, sends supported filters directly to the underlying database. +It can be disabled with `--disable-filter-pushdown`. +The **Sort Pushdown**, which can be enabled with `--enable-sort-pushdown`, pushes down sorting, `limit`, and `skip` methods. + +There might be some cases, though, where FerretDB needs to fetch more data to make sure +that it'll apply methods such as sorting correctly, as for some parts of the query pushdown cannot be applied. +If you have a need to sort pushdown anyway, you can enable **Unsafe Sort Pushdown** with `--enable-unsafe-sort-pushdown`. + ## Supported types and operators The following table shows all operators and types that FerretDB pushdowns on PostgreSQL backend. diff --git a/website/docs/reference/glossary.md b/website/docs/reference/glossary.md index 36acaf43266e..4d1fc45745e9 100644 --- a/website/docs/reference/glossary.md +++ b/website/docs/reference/glossary.md @@ -146,6 +146,12 @@ See [Operation modes](../configuration/operation-modes.md) for more details. An open source relational database. FerretDB uses PostgreSQL as a database engine. +#### Pushdown + +The method of optimizing a query by reducing the amount of data read and processed in FerretDB, by moving some logic +to underlying backend. +See [Query pushdown](../pushdown.md) for more details. + ### S #### SQLite From bedd581e2c49154131ae2fb1bbc417119fa6a804 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 20:15:32 +0100 Subject: [PATCH 16/25] wip --- .../aggregate_documents_compat_test.go | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/integration/aggregate_documents_compat_test.go b/integration/aggregate_documents_compat_test.go index 76597881c735..6e6af7449566 100644 --- a/integration/aggregate_documents_compat_test.go +++ b/integration/aggregate_documents_compat_test.go @@ -134,27 +134,25 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P nonEmptyResults = true } - if targetErr == nil { - explainCommand := bson.D{{"explain", bson.D{ - {"aggregate", targetCollection.Name()}, - {"pipeline", pipeline}, - }}} - var explainRes bson.D - require.NoError(t, targetCollection.Database().RunCommand(ctx, explainCommand).Decode(&explainRes)) - - resultPushdown := tc.resultPushdown - - var msg string - // TODO https://github.com/FerretDB/FerretDB/issues/3386 - if setup.FilterPushdownDisabled() { - resultPushdown = noPushdown - msg = "Fitler pushdown is disabled, but target resulted with pushdown" - } - - doc := ConvertDocument(t, explainRes) - pushdown, _ := doc.Get("pushdown") - assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) + explainCommand := bson.D{{"explain", bson.D{ + {"aggregate", targetCollection.Name()}, + {"pipeline", pipeline}, + }}} + var explainRes bson.D + require.NoError(t, targetCollection.Database().RunCommand(ctx, explainCommand).Decode(&explainRes)) + + resultPushdown := tc.resultPushdown + + var msg string + // TODO https://github.com/FerretDB/FerretDB/issues/3386 + if setup.FilterPushdownDisabled() { + resultPushdown = noPushdown + msg = "Fitler pushdown is disabled, but target resulted with pushdown" } + + doc := ConvertDocument(t, explainRes) + pushdown, _ := doc.Get("pushdown") + assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) }) } From f913767b6ee6405a10285c5223101540a517e625 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 15 Nov 2023 20:16:21 +0100 Subject: [PATCH 17/25] wip --- integration/aggregate_documents_compat_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/aggregate_documents_compat_test.go b/integration/aggregate_documents_compat_test.go index 6e6af7449566..74a3f3703957 100644 --- a/integration/aggregate_documents_compat_test.go +++ b/integration/aggregate_documents_compat_test.go @@ -141,18 +141,18 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P var explainRes bson.D require.NoError(t, targetCollection.Database().RunCommand(ctx, explainCommand).Decode(&explainRes)) - resultPushdown := tc.resultPushdown + resPushdown := tc.resultPushdown var msg string // TODO https://github.com/FerretDB/FerretDB/issues/3386 if setup.FilterPushdownDisabled() { - resultPushdown = noPushdown + resPushdown = noPushdown msg = "Fitler pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, explainRes) pushdown, _ := doc.Get("pushdown") - assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) + assert.Equal(t, resPushdown.FilterPushdownExpected(t), pushdown, msg) }) } From 2adeed3b62f118ebeb12f7e2c6fb3798bc3ebf9b Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 16 Nov 2023 13:08:49 +0100 Subject: [PATCH 18/25] split --- Taskfile.yml | 4 ---- cmd/ferretdb/main.go | 2 -- integration/setup/listener.go | 5 ----- integration/setup/setup.go | 1 - integration/setup/test_helpers.go | 6 +++--- internal/handlers/registry/hana.go | 1 - internal/handlers/registry/postgresql.go | 1 - internal/handlers/registry/registry.go | 1 - internal/handlers/registry/sqlite.go | 1 - internal/handlers/sqlite/msg_aggregate.go | 2 +- internal/handlers/sqlite/msg_explain.go | 10 ++++------ internal/handlers/sqlite/msg_find.go | 8 +++----- internal/handlers/sqlite/sqlite.go | 1 - 13 files changed, 11 insertions(+), 32 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index c0e2631b6de8..5b7897ef16f5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,7 +9,6 @@ vars: SHARD_INDEX: 1 SHARD_TOTAL: 1 DISABLE_FILTER_PUSHDOWN: false - ENABLE_SORT_PUSHDOWN: false ENABLE_UNSAFE_SORT_PUSHDOWN: false TEST_RUN: "" TEST_TIMEOUT: 35m @@ -219,7 +218,6 @@ tasks: -postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?search_path=' -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-sqlite: @@ -246,7 +244,6 @@ tasks: -target-tls -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-hana: @@ -273,7 +270,6 @@ tasks: -hana-url=$FERRETDB_HANA_URL -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown={{.DISABLE_FILTER_PUSHDOWN}} - -enable-sort-pushdown={{.ENABLE_SORT_PUSHDOWN}} -enable-unsafe-sort-pushdown={{.ENABLE_UNSAFE_SORT_PUSHDOWN}} test-integration-mongodb: diff --git a/cmd/ferretdb/main.go b/cmd/ferretdb/main.go index 1937e42f0cac..8b215150a5fd 100644 --- a/cmd/ferretdb/main.go +++ b/cmd/ferretdb/main.go @@ -87,7 +87,6 @@ var cli struct { RecordsDir string `default:"" help:"Testing: directory for record files."` DisableFilterPushdown bool `default:"false" help:"Experimental: disable filter pushdown."` - EnableSortPushdown bool `default:"false" help:"Experimental: enable sort pushdown."` EnableUnsafeSortPushdown bool `default:"false" help:"Experimental: enable unsafe sort pushdown."` EnableOplog bool `default:"false" help:"Experimental: enable capped collections, tailable cursors and OpLog." hidden:""` @@ -383,7 +382,6 @@ func run() { TestOpts: registry.TestOpts{ DisableFilterPushdown: cli.Test.DisableFilterPushdown, - EnableSortPushdown: cli.Test.EnableSortPushdown, EnableUnsafeSortPushdown: cli.Test.EnableUnsafeSortPushdown, EnableOplog: cli.Test.EnableOplog, }, diff --git a/integration/setup/listener.go b/integration/setup/listener.go index f5bb42044e93..dddaacda86bf 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -160,7 +160,6 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string TestOpts: registry.TestOpts{ DisableFilterPushdown: *disableFilterPushdownF, - EnableSortPushdown: *enableSortPushdownF, EnableUnsafeSortPushdown: *enableUnsafeSortPushdownF, EnableOplog: true, }, @@ -187,10 +186,6 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string tb.Fatal("Both -target-tls and -target-unix-socket are set.") } - if *enableSortPushdownF && *enableUnsafeSortPushdownF { - tb.Fatal("Both -enable-sort-pushdown and -enable-unsafe-sort-pushdown are set.") - } - switch { case *targetTLSF: listenerOpts.TLS = "127.0.0.1:0" diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 98d0d2737989..001ccbd5fff3 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -60,7 +60,6 @@ var ( logLevelF = zap.LevelFlag("log-level", zap.DebugLevel, "log level for tests") disableFilterPushdownF = flag.Bool("disable-filter-pushdown", false, "disable filter pushdown") - enableSortPushdownF = flag.Bool("enable-sort-pushdown", false, "enable sort pushdown") enableUnsafeSortPushdownF = flag.Bool("enable-unsafe-sort-pushdown", false, "enable unsafe sort pushdown") ) diff --git a/integration/setup/test_helpers.go b/integration/setup/test_helpers.go index c08ea4af3ade..8c25202fef4c 100644 --- a/integration/setup/test_helpers.go +++ b/integration/setup/test_helpers.go @@ -86,7 +86,7 @@ func FilterPushdownDisabled() bool { return *disableFilterPushdownF } -// SortPushdownEnabled returns true if sort pushdown is enabled. -func SortPushdownEnabled() bool { - return *enableSortPushdownF +// UnsafeSortPushdownEnabled returns true if unsafe sort pushdown is enabled. +func UnsafeSortPushdownEnabled() bool { + return *enableUnsafeSortPushdownF } diff --git a/internal/handlers/registry/hana.go b/internal/handlers/registry/hana.go index 8caa7aa8e5ae..773c32fd7949 100644 --- a/internal/handlers/registry/hana.go +++ b/internal/handlers/registry/hana.go @@ -44,7 +44,6 @@ func init() { StateProvider: opts.StateProvider, DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/registry/postgresql.go b/internal/handlers/registry/postgresql.go index 3370cf95eff9..c44b5ef56d99 100644 --- a/internal/handlers/registry/postgresql.go +++ b/internal/handlers/registry/postgresql.go @@ -40,7 +40,6 @@ func init() { StateProvider: opts.StateProvider, DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index 44661b4ce926..73ecb28fe92a 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -59,7 +59,6 @@ type NewHandlerOpts struct { // TestOpts represents experimental configuration options. type TestOpts struct { DisableFilterPushdown bool - EnableSortPushdown bool EnableUnsafeSortPushdown bool EnableOplog bool } diff --git a/internal/handlers/registry/sqlite.go b/internal/handlers/registry/sqlite.go index 0753567172e0..77a7cfe2348e 100644 --- a/internal/handlers/registry/sqlite.go +++ b/internal/handlers/registry/sqlite.go @@ -40,7 +40,6 @@ func init() { StateProvider: opts.StateProvider, DisableFilterPushdown: opts.DisableFilterPushdown, - EnableSortPushdown: opts.EnableSortPushdown, EnableUnsafeSortPushdown: opts.EnableUnsafeSortPushdown, EnableOplog: opts.EnableOplog, } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 5d841e5a4a52..4c84d02b602a 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -269,7 +269,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs } // Skip sorting if there are more than one sort parameters - if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && sort.Len() == 1 { + if h.EnableUnsafeSortPushdown && sort.Len() == 1 { var order types.SortType k := sort.Keys()[0] diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 2944920cbaff..2b7e8eaec72d 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -88,7 +88,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } // Skip sorting if there are more than one sort parameters - if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Sort.Len() == 1 { + if h.EnableUnsafeSortPushdown && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -107,12 +107,10 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but sort pushdown is not enabled, it must fetch all documents + // - `sort` is set but `UnsafeSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && - (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && - params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } @@ -120,7 +118,7 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, qp.Filter = nil } - if !h.EnableSortPushdown { + if !h.EnableUnsafeSortPushdown { qp.Sort = nil } diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index e8abb9e734cd..596f10a1798d 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -113,7 +113,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er } // Skip sorting if there are more than one sort parameters - if (h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && params.Sort.Len() == 1 { + if h.EnableUnsafeSortPushdown && params.Sort.Len() == 1 { var order types.SortType k := params.Sort.Keys()[0] @@ -132,12 +132,10 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // Limit pushdown is not applied if: // - `filter` is set, it must fetch all documents to filter them in memory; - // - `sort` is set but sort pushdown is not enabled, it must fetch all documents + // - `sort` is set but `UnsafeSortPushdown` is not set, it must fetch all documents // and sort them in memory; // - `skip` is non-zero value, skip pushdown is not supported yet. - if params.Filter.Len() == 0 && - (params.Sort.Len() == 0 || h.EnableSortPushdown || h.EnableUnsafeSortPushdown) && - params.Skip == 0 { + if params.Filter.Len() == 0 && (params.Sort.Len() == 0 || h.EnableUnsafeSortPushdown) && params.Skip == 0 { qp.Limit = params.Limit } diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 7369d6099871..80a2984672e7 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -50,7 +50,6 @@ type NewOpts struct { // test options DisableFilterPushdown bool - EnableSortPushdown bool EnableUnsafeSortPushdown bool EnableOplog bool } From 8f901f401818e392ec26a9acfdc296f3b2763acd Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 16 Nov 2023 13:13:21 +0100 Subject: [PATCH 19/25] cleanup --- integration/pushdown.go | 2 +- integration/query_test.go | 2 +- website/docs/pushdown.md | 12 ------------ 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/integration/pushdown.go b/integration/pushdown.go index 90edeb7e3df2..21a4cb2492a5 100644 --- a/integration/pushdown.go +++ b/integration/pushdown.go @@ -58,7 +58,7 @@ func (res resultPushdown) FilterPushdownExpected(t testtb.TB) bool { // It checks if pushdown is enabled by flag. // For capped collection, pushdown for recordID is done even if pushdown is not enabled by flag. func (res resultPushdown) SortPushdownExpected(t testtb.TB, cappedCollection bool) bool { - if !setup.SortPushdownEnabled() && cappedCollection { + if !setup.UnsafeSortPushdownEnabled() && cappedCollection { res = allPushdown } diff --git a/integration/query_test.go b/integration/query_test.go index 17631521fb77..0856c63241fb 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -995,7 +995,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { var msg string - if !setup.SortPushdownEnabled() && tc.sort != nil { + if !setup.UnsafeSortPushdownEnabled() && tc.sort != nil { tc.limitPushdown = false msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } diff --git a/website/docs/pushdown.md b/website/docs/pushdown.md index 281d94a37616..c6f0f228cfcf 100644 --- a/website/docs/pushdown.md +++ b/website/docs/pushdown.md @@ -17,18 +17,6 @@ To make this process more efficient, we minimize the amount of incoming data, by You can learn more about query pushdown in our [blog post](https://blog.ferretdb.io/ferretdb-fetches-data-query-pushdown/). ::: -## Types of pushdown - -Currently we support 3 different types of pushdown. - -The **Filter pushdown**, which is enabled by default, sends supported filters directly to the underlying database. -It can be disabled with `--disable-filter-pushdown`. -The **Sort Pushdown**, which can be enabled with `--enable-sort-pushdown`, pushes down sorting, `limit`, and `skip` methods. - -There might be some cases, though, where FerretDB needs to fetch more data to make sure -that it'll apply methods such as sorting correctly, as for some parts of the query pushdown cannot be applied. -If you have a need to sort pushdown anyway, you can enable **Unsafe Sort Pushdown** with `--enable-unsafe-sort-pushdown`. - ## Supported types and operators The following table shows all operators and types that FerretDB pushdowns on PostgreSQL backend. From a7e17e2e09709d7ffd88c45b5c138ea363d326ee Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 12:59:09 +0100 Subject: [PATCH 20/25] rename --- integration/query_command_compat_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/query_command_compat_test.go b/integration/query_command_compat_test.go index c2ffdf0f5e63..1279e06528b8 100644 --- a/integration/query_command_compat_test.go +++ b/integration/query_command_compat_test.go @@ -37,7 +37,7 @@ type queryCommandCompatTestCase struct { optSkip any // defaults to nil to leave unset limit *int64 // defaults to nil to leave unset resultType compatTestCaseResultType // defaults to nonEmptyResult - resultPushdown resultPushdown // defaults to noPushdown + filterPushdown resultPushdown // defaults to noPushdown skip string // skip test for all handlers, must have issue number mentioned } @@ -103,13 +103,13 @@ func testQueryCommandCompat(t *testing.T, testCases map[string]queryCommandCompa var msg string if setup.FilterPushdownDisabled() { - tc.resultPushdown = noPushdown + tc.filterPushdown = noPushdown msg = "Filter pushdown is disabled, but target resulted with pushdown" } doc := ConvertDocument(t, explainRes) pushdown, _ := doc.Get("pushdown") - assert.Equal(t, tc.resultPushdown.FilterPushdownExpected(t), pushdown, msg) + assert.Equal(t, tc.filterPushdown.FilterPushdownExpected(t), pushdown, msg) targetCommand := append( bson.D{ From 52568d81fd8f31cc3cdf49abcf78eb12ed53d14e Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 13:14:29 +0100 Subject: [PATCH 21/25] rename --- integration/query_test.go | 46 +++++++++++-------------- internal/handlers/sqlite/msg_explain.go | 4 +-- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 0856c63241fb..724ba3938163 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -836,7 +836,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { len int // expected length of results filterPushdown resultPushdown // optional, defaults to noPushdown - limitPushdown bool // optional, set true for expected pushdown for limit + limitPushdown resultPushdown // optional, defaults to noPushdown err *mongo.CommandError // optional, expected error from MongoDB altMessage string // optional, alternative error message for FerretDB, ignored if empty skip string // optional, skip test with a specified reason @@ -845,39 +845,39 @@ func TestQueryCommandLimitPushDown(t *testing.T) { "Simple": { limit: 1, len: 1, - limitPushdown: true, + limitPushdown: pgPushdown, }, "AlmostAll": { limit: int64(len(shareddata.Composites.Docs()) - 1), len: len(shareddata.Composites.Docs()) - 1, - limitPushdown: true, + limitPushdown: pgPushdown, }, "All": { limit: int64(len(shareddata.Composites.Docs())), len: len(shareddata.Composites.Docs()), - limitPushdown: true, + limitPushdown: pgPushdown, }, "More": { limit: int64(len(shareddata.Composites.Docs()) + 1), len: len(shareddata.Composites.Docs()), - limitPushdown: true, + limitPushdown: pgPushdown, }, "Big": { limit: 1000, len: len(shareddata.Composites.Docs()), - limitPushdown: true, + limitPushdown: pgPushdown, }, "Zero": { limit: 0, len: len(shareddata.Composites.Docs()), - limitPushdown: false, + limitPushdown: noPushdown, }, "IDFilter": { filter: bson.D{{"_id", "array"}}, limit: 3, len: 1, filterPushdown: allPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "ValueFilter": { filter: bson.D{{"v", 42}}, @@ -885,28 +885,28 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 3, len: 3, filterPushdown: pgPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "DotNotationFilter": { filter: bson.D{{"v.foo", 42}}, limit: 3, len: 3, filterPushdown: noPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "ObjectFilter": { filter: bson.D{{"v", bson.D{{"foo", nil}}}}, limit: 3, len: 1, filterPushdown: noPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "Sort": { sort: bson.D{{"_id", 1}}, limit: 2, len: 2, filterPushdown: noPushdown, - limitPushdown: true, + limitPushdown: pgPushdown, }, "IDFilterSort": { filter: bson.D{{"_id", "array"}}, @@ -914,7 +914,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 3, len: 1, filterPushdown: allPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "ValueFilterSort": { filter: bson.D{{"v", 42}}, @@ -922,7 +922,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 3, len: 3, filterPushdown: pgPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "DotNotationFilterSort": { filter: bson.D{{"v.foo", 42}}, @@ -930,7 +930,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 3, len: 3, filterPushdown: noPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "ObjectFilterSort": { filter: bson.D{{"v", bson.D{{"foo", nil}}}}, @@ -938,13 +938,13 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 3, len: 1, filterPushdown: noPushdown, - limitPushdown: false, + limitPushdown: noPushdown, }, "Skip": { optSkip: pointer.ToInt64(1), limit: 2, len: 2, - limitPushdown: false, + limitPushdown: noPushdown, }, } { tc, name := tc, name @@ -996,23 +996,19 @@ func TestQueryCommandLimitPushDown(t *testing.T) { var msg string if !setup.UnsafeSortPushdownEnabled() && tc.sort != nil { - tc.limitPushdown = false msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } - resultPushdown := tc.filterPushdown + doc := ConvertDocument(t, res) + limitPushdown, _ := doc.Get("limitPushdown") + assert.Equal(t, tc.limitPushdown.SortPushdownExpected(t, false), limitPushdown, msg) if setup.FilterPushdownDisabled() { - resultPushdown = noPushdown msg = "Filter pushdown is disabled, but target resulted with pushdown" } - doc := ConvertDocument(t, res) - limitPushdown, _ := doc.Get("limitPushdown") - assert.Equal(t, tc.limitPushdown, limitPushdown, msg) - filterPushdown, _ := ConvertDocument(t, res).Get("pushdown") - assert.Equal(t, resultPushdown.FilterPushdownExpected(t), filterPushdown, msg) + assert.Equal(t, tc.filterPushdown.FilterPushdownExpected(t), filterPushdown, msg) }) t.Run("Find", func(t *testing.T) { diff --git a/internal/handlers/sqlite/msg_explain.go b/internal/handlers/sqlite/msg_explain.go index 2b7e8eaec72d..dd0ab4b979b3 100644 --- a/internal/handlers/sqlite/msg_explain.go +++ b/internal/handlers/sqlite/msg_explain.go @@ -137,8 +137,8 @@ func (h *Handler) MsgExplain(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // our extensions // TODO https://github.com/FerretDB/FerretDB/issues/3235 - "pushdown", res.FilterPushdown, - "sortingPushdown", res.SortPushdown, + "filterPushdown", res.FilterPushdown, + "sortPushdown", res.SortPushdown, "limitPushdown", res.LimitPushdown, "ok", float64(1), From 5aaf0c8c5384b904b8ceabd6302ae6cc97fc1643 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 13:25:42 +0100 Subject: [PATCH 22/25] rm docs --- website/docs/reference/glossary.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/website/docs/reference/glossary.md b/website/docs/reference/glossary.md index 4d1fc45745e9..36acaf43266e 100644 --- a/website/docs/reference/glossary.md +++ b/website/docs/reference/glossary.md @@ -146,12 +146,6 @@ See [Operation modes](../configuration/operation-modes.md) for more details. An open source relational database. FerretDB uses PostgreSQL as a database engine. -#### Pushdown - -The method of optimizing a query by reducing the amount of data read and processed in FerretDB, by moving some logic -to underlying backend. -See [Query pushdown](../pushdown.md) for more details. - ### S #### SQLite From 68db116638da0655fb3bc8ab175aa517fb04c3cc Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 15:59:18 +0100 Subject: [PATCH 23/25] fix --- integration/aggregate_documents_compat_test.go | 2 +- integration/query_command_compat_test.go | 2 +- integration/query_compat_test.go | 4 ++-- integration/query_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration/aggregate_documents_compat_test.go b/integration/aggregate_documents_compat_test.go index 74a3f3703957..ae2fef21783b 100644 --- a/integration/aggregate_documents_compat_test.go +++ b/integration/aggregate_documents_compat_test.go @@ -151,7 +151,7 @@ func testAggregateStagesCompatWithProviders(t *testing.T, providers shareddata.P } doc := ConvertDocument(t, explainRes) - pushdown, _ := doc.Get("pushdown") + pushdown, _ := doc.Get("filterPushdown") assert.Equal(t, resPushdown.FilterPushdownExpected(t), pushdown, msg) }) } diff --git a/integration/query_command_compat_test.go b/integration/query_command_compat_test.go index 1279e06528b8..ba24403cd624 100644 --- a/integration/query_command_compat_test.go +++ b/integration/query_command_compat_test.go @@ -108,7 +108,7 @@ func testQueryCommandCompat(t *testing.T, testCases map[string]queryCommandCompa } doc := ConvertDocument(t, explainRes) - pushdown, _ := doc.Get("pushdown") + pushdown, _ := doc.Get("filterPushdown") assert.Equal(t, tc.filterPushdown.FilterPushdownExpected(t), pushdown, msg) targetCommand := append( diff --git a/integration/query_compat_test.go b/integration/query_compat_test.go index 5f59dd3ee327..93975fb693b5 100644 --- a/integration/query_compat_test.go +++ b/integration/query_compat_test.go @@ -137,7 +137,7 @@ func testQueryCompatWithProviders(t *testing.T, providers shareddata.Providers, } doc := ConvertDocument(t, explainRes) - pushdown, _ := doc.Get("pushdown") + pushdown, _ := doc.Get("filterPushdown") assert.Equal(t, resultPushdown.FilterPushdownExpected(t), pushdown, msg) targetCursor, targetErr := targetCollection.Find(ctx, filter, opts) @@ -288,7 +288,7 @@ func TestQueryCappedCollectionCompat(t *testing.T) { require.NoError(t, targetCollection.Database().RunCommand(ctx, bson.D{{"explain", explainQuery}}).Decode(&explainRes)) doc := ConvertDocument(t, explainRes) - sortPushdown, _ := doc.Get("sortingPushdown") + sortPushdown, _ := doc.Get("sortPushdown") assert.Equal(t, tc.sortPushdown.SortPushdownExpected(t, true), sortPushdown) findOpts := options.Find() diff --git a/integration/query_test.go b/integration/query_test.go index 724ba3938163..f8829aca8e2e 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -1007,7 +1007,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { msg = "Filter pushdown is disabled, but target resulted with pushdown" } - filterPushdown, _ := ConvertDocument(t, res).Get("pushdown") + filterPushdown, _ := ConvertDocument(t, res).Get("filterPushdown") assert.Equal(t, tc.filterPushdown.FilterPushdownExpected(t), filterPushdown, msg) }) From 2016d9be2cccb53ad2f6fd063689b48d2da609c7 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 16:17:54 +0100 Subject: [PATCH 24/25] fix --- integration/query_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/query_test.go b/integration/query_test.go index f8829aca8e2e..7e0e8aaadfac 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -996,6 +996,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { var msg string if !setup.UnsafeSortPushdownEnabled() && tc.sort != nil { + tc.limitPushdown = noPushdown msg = "Sort pushdown is disabled, but target resulted with limitPushdown" } @@ -1004,6 +1005,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { assert.Equal(t, tc.limitPushdown.SortPushdownExpected(t, false), limitPushdown, msg) if setup.FilterPushdownDisabled() { + tc.filterPushdown = noPushdown msg = "Filter pushdown is disabled, but target resulted with pushdown" } From 2852ea970656ff6e73ceb994898690d4ca146205 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 17 Nov 2023 16:32:17 +0100 Subject: [PATCH 25/25] fix --- integration/query_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/query_test.go b/integration/query_test.go index 7e0e8aaadfac..e763ed820c31 100644 --- a/integration/query_test.go +++ b/integration/query_test.go @@ -845,27 +845,27 @@ func TestQueryCommandLimitPushDown(t *testing.T) { "Simple": { limit: 1, len: 1, - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "AlmostAll": { limit: int64(len(shareddata.Composites.Docs()) - 1), len: len(shareddata.Composites.Docs()) - 1, - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "All": { limit: int64(len(shareddata.Composites.Docs())), len: len(shareddata.Composites.Docs()), - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "More": { limit: int64(len(shareddata.Composites.Docs()) + 1), len: len(shareddata.Composites.Docs()), - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "Big": { limit: 1000, len: len(shareddata.Composites.Docs()), - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "Zero": { limit: 0, @@ -906,7 +906,7 @@ func TestQueryCommandLimitPushDown(t *testing.T) { limit: 2, len: 2, filterPushdown: noPushdown, - limitPushdown: pgPushdown, + limitPushdown: allPushdown, }, "IDFilterSort": { filter: bson.D{{"_id", "array"}},