From f5788f2e3fccdd328047363e8e11b33603204180 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 07:24:57 +0200 Subject: [PATCH 01/21] fix --- internal/backends/postgresql/collection.go | 52 +++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 45acad58ed2f..b9c96623b079 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -28,6 +28,8 @@ import ( "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool" "github.com/FerretDB/FerretDB/internal/handlers/sjson" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" ) @@ -245,8 +247,54 @@ func (c *collection) DeleteAll(ctx context.Context, params *backends.DeleteAllPa // Explain implements backends.Collection interface. func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams) (*backends.ExplainResult, error) { - // TODO https://github.com/FerretDB/FerretDB/issues/3389 - return new(backends.ExplainResult), nil + p, err := c.r.DatabaseGetExisting(ctx, c.dbName) + if err != nil { + return nil, lazyerrors.Error(err) + } + + if p == nil { + return &backends.ExplainResult{}, nil + } + + meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) + if err != nil { + return nil, lazyerrors.Error(err) + } + + if meta == nil { + return &backends.ExplainResult{ + QueryPlanner: must.NotFail(types.NewDocument()), + }, nil + } + + // TODO https://github.com/FerretDB/FerretDB/issues/3414 + q := fmt.Sprintf( + `EXPLAIN (VERBOSE true, FORMAT JSON) SELECT %s FROM %s`, + metadata.DefaultColumn, + pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), + ) + + rows, err := p.Query(ctx, q) + if err != nil { + return nil, lazyerrors.Error(err) + } + + iter := newQueryIterator(ctx, rows) + defer iter.Close() + + _, queryPlan, err := iter.Next() + if err != nil { + return nil, lazyerrors.Error(err) + } + + _, _, err = iter.Next() + if err != iterator.ErrIteratorDone { + return nil, lazyerrors.Errorf("pg explain iterator is not done yet, while it should, (err: %v), expected: %v", err, iterator.ErrIteratorDone) + } + + return &backends.ExplainResult{ + QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), + }, nil } // Stats implements backends.Collection interface. From 72f2e1d42adff9d2ac2cc3e675f80c7c86e75c6a Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 07:54:29 +0200 Subject: [PATCH 02/21] test --- internal/backends/postgresql/collection.go | 64 ++++++++++++++++--- .../backends/postgresql/query_iterator.go | 17 +++-- internal/handlers/registry/registry.go | 2 + 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index b9c96623b079..d51f8190be9c 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -16,6 +16,7 @@ package postgresql import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -23,13 +24,13 @@ import ( "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + "golang.org/x/exp/maps" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool" "github.com/FerretDB/FerretDB/internal/handlers/sjson" "github.com/FerretDB/FerretDB/internal/types" - "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" ) @@ -59,7 +60,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if p == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, nil), + Iter: newQueryIterator(ctx, nil, nil), }, nil } @@ -70,7 +71,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if meta == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, nil), + Iter: newQueryIterator(ctx, nil, nil), }, nil } @@ -87,7 +88,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* } return &backends.QueryResult{ - Iter: newQueryIterator(ctx, rows), + Iter: newQueryIterator(ctx, rows, nil), }, nil } @@ -279,7 +280,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - iter := newQueryIterator(ctx, rows) + iter := newQueryIterator(ctx, rows, unmarshalExplain) defer iter.Close() _, queryPlan, err := iter.Next() @@ -287,16 +288,61 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - _, _, err = iter.Next() - if err != iterator.ErrIteratorDone { - return nil, lazyerrors.Errorf("pg explain iterator is not done yet, while it should, (err: %v), expected: %v", err, iterator.ErrIteratorDone) - } + // _, _, err = iter.Next() + // if err != iterator.ErrIteratorDone { + // return nil, lazyerrors.Errorf("pg explain iterator is not done yet, while it should, (err: %v), expected: %v", err, iterator.ErrIteratorDone) + // } return &backends.ExplainResult{ QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), }, nil } +// unmarshalExplain unmarshalls the plan from EXPLAIN postgreSQL command. +// EXPLAIN result is not sjson, so it cannot be unmarshalled by sjson.Unmarshal. +func unmarshalExplain(b []byte) (*types.Document, error) { + var plans []map[string]any + if err := json.Unmarshal(b, &plans); err != nil { + return nil, lazyerrors.Error(err) + } + + if len(plans) == 0 { + return nil, lazyerrors.Error(errors.New("no execution plan returned")) + } + + return convertJSON(plans[0]).(*types.Document), nil +} + +// convertJSON transforms decoded JSON map[string]any value into *types.Document. +func convertJSON(value any) any { + switch value := value.(type) { + case map[string]any: + d := types.MakeDocument(len(value)) + keys := maps.Keys(value) + for _, k := range keys { + v := value[k] + d.Set(k, convertJSON(v)) + } + return d + + case []any: + a := types.MakeArray(len(value)) + for _, v := range value { + a.Append(convertJSON(v)) + } + return a + + case nil: + return types.Null + + case float64, string, bool: + return value + + default: + panic(fmt.Sprintf("unsupported type: %[1]T (%[1]v)", value)) + } +} + // Stats implements backends.Collection interface. func (c *collection) Stats(ctx context.Context, params *backends.CollectionStatsParams) (*backends.CollectionStatsResult, error) { // TODO https://github.com/FerretDB/FerretDB/issues/3398 diff --git a/internal/backends/postgresql/query_iterator.go b/internal/backends/postgresql/query_iterator.go index 813619cd16b5..0e41c55e974b 100644 --- a/internal/backends/postgresql/query_iterator.go +++ b/internal/backends/postgresql/query_iterator.go @@ -32,6 +32,8 @@ import ( type queryIterator struct { // the order of fields is weird to make the struct smaller due to alignment + unmarshal func(b []byte) (*types.Document, error) // defaults to sjson.Unmarshal + ctx context.Context rows pgx.Rows // protected by m token *resource.Token @@ -47,11 +49,16 @@ type queryIterator struct { // // Nil rows are possible and return already done iterator. // It still should be Close'd. -func newQueryIterator(ctx context.Context, rows pgx.Rows) types.DocumentsIterator { +func newQueryIterator(ctx context.Context, rows pgx.Rows, unmarshal func(b []byte) (*types.Document, error)) types.DocumentsIterator { + if unmarshal == nil { + unmarshal = sjson.Unmarshal + } + iter := &queryIterator{ - ctx: ctx, - rows: rows, - token: resource.NewToken(), + ctx: ctx, + rows: rows, + token: resource.NewToken(), + unmarshal: unmarshal, } resource.Track(iter, iter.token) @@ -95,7 +102,7 @@ func (iter *queryIterator) Next() (struct{}, *types.Document, error) { return unused, nil, lazyerrors.Error(err) } - doc, err := sjson.Unmarshal(b) + doc, err := iter.unmarshal(b) if err != nil { iter.close() return unused, nil, lazyerrors.Error(err) diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index 7c0babed6531..d9769ca7a92b 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -69,6 +69,8 @@ func NewHandler(name string, opts *NewHandlerOpts) (handlers.Interface, error) { return nil, fmt.Errorf("opts is nil") } + opts.UseNewPG = true + newHandler := registry[name] if newHandler == nil { return nil, fmt.Errorf("unknown handler %q", name) From f6e36b0f128dbbc14b2f69ceb7f28964a5681072 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 08:11:12 +0200 Subject: [PATCH 03/21] wip --- internal/backends/postgresql/collection.go | 52 ---------------- internal/backends/postgresql/helpers.go | 70 ++++++++++++++++++++++ 2 files changed, 70 insertions(+), 52 deletions(-) create mode 100644 internal/backends/postgresql/helpers.go diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index d51f8190be9c..1a51c715b71e 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -16,7 +16,6 @@ package postgresql import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -24,7 +23,6 @@ import ( "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" - "golang.org/x/exp/maps" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" @@ -288,61 +286,11 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - // _, _, err = iter.Next() - // if err != iterator.ErrIteratorDone { - // return nil, lazyerrors.Errorf("pg explain iterator is not done yet, while it should, (err: %v), expected: %v", err, iterator.ErrIteratorDone) - // } - return &backends.ExplainResult{ QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), }, nil } -// unmarshalExplain unmarshalls the plan from EXPLAIN postgreSQL command. -// EXPLAIN result is not sjson, so it cannot be unmarshalled by sjson.Unmarshal. -func unmarshalExplain(b []byte) (*types.Document, error) { - var plans []map[string]any - if err := json.Unmarshal(b, &plans); err != nil { - return nil, lazyerrors.Error(err) - } - - if len(plans) == 0 { - return nil, lazyerrors.Error(errors.New("no execution plan returned")) - } - - return convertJSON(plans[0]).(*types.Document), nil -} - -// convertJSON transforms decoded JSON map[string]any value into *types.Document. -func convertJSON(value any) any { - switch value := value.(type) { - case map[string]any: - d := types.MakeDocument(len(value)) - keys := maps.Keys(value) - for _, k := range keys { - v := value[k] - d.Set(k, convertJSON(v)) - } - return d - - case []any: - a := types.MakeArray(len(value)) - for _, v := range value { - a.Append(convertJSON(v)) - } - return a - - case nil: - return types.Null - - case float64, string, bool: - return value - - default: - panic(fmt.Sprintf("unsupported type: %[1]T (%[1]v)", value)) - } -} - // Stats implements backends.Collection interface. func (c *collection) Stats(ctx context.Context, params *backends.CollectionStatsParams) (*backends.CollectionStatsResult, error) { // TODO https://github.com/FerretDB/FerretDB/issues/3398 diff --git a/internal/backends/postgresql/helpers.go b/internal/backends/postgresql/helpers.go new file mode 100644 index 000000000000..21bdc4c6f306 --- /dev/null +++ b/internal/backends/postgresql/helpers.go @@ -0,0 +1,70 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgresql + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" + "golang.org/x/exp/maps" +) + +// unmarshalExplain unmarshalls the plan from EXPLAIN postgreSQL command. +// EXPLAIN result is not sjson, so it cannot be unmarshalled by sjson.Unmarshal. +func unmarshalExplain(b []byte) (*types.Document, error) { + var plans []map[string]any + if err := json.Unmarshal(b, &plans); err != nil { + return nil, lazyerrors.Error(err) + } + + if len(plans) == 0 { + return nil, lazyerrors.Error(errors.New("no execution plan returned")) + } + + return convertJSON(plans[0]).(*types.Document), nil +} + +// convertJSON transforms decoded JSON map[string]any value into *types.Document. +func convertJSON(value any) any { + switch value := value.(type) { + case map[string]any: + d := types.MakeDocument(len(value)) + keys := maps.Keys(value) + for _, k := range keys { + v := value[k] + d.Set(k, convertJSON(v)) + } + return d + + case []any: + a := types.MakeArray(len(value)) + for _, v := range value { + a.Append(convertJSON(v)) + } + return a + + case nil: + return types.Null + + case float64, string, bool: + return value + + default: + panic(fmt.Sprintf("unsupported type: %[1]T (%[1]v)", value)) + } +} From 51415024ec402c47b2c9ee2f5e5c47a19e036631 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 08:11:36 +0200 Subject: [PATCH 04/21] wip --- internal/handlers/registry/registry.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/handlers/registry/registry.go b/internal/handlers/registry/registry.go index d9769ca7a92b..7c0babed6531 100644 --- a/internal/handlers/registry/registry.go +++ b/internal/handlers/registry/registry.go @@ -69,8 +69,6 @@ func NewHandler(name string, opts *NewHandlerOpts) (handlers.Interface, error) { return nil, fmt.Errorf("opts is nil") } - opts.UseNewPG = true - newHandler := registry[name] if newHandler == nil { return nil, fmt.Errorf("unknown handler %q", name) From 0d681c9d853980714023c947649589ac56d0c8c0 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 13:38:30 +0200 Subject: [PATCH 05/21] fmt --- internal/backends/postgresql/helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/backends/postgresql/helpers.go b/internal/backends/postgresql/helpers.go index 21bdc4c6f306..fe3824287830 100644 --- a/internal/backends/postgresql/helpers.go +++ b/internal/backends/postgresql/helpers.go @@ -19,9 +19,10 @@ import ( "errors" "fmt" + "golang.org/x/exp/maps" + "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" - "golang.org/x/exp/maps" ) // unmarshalExplain unmarshalls the plan from EXPLAIN postgreSQL command. From 2dd28582961ff03c4b5a338e812c757ba85e57aa Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 14:13:17 +0200 Subject: [PATCH 06/21] lint --- internal/backends/postgresql/collection.go | 14 ++++++++++---- internal/backends/postgresql/query_iterator.go | 16 +++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 1a51c715b71e..d3d3877d4802 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -58,7 +58,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if p == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, nil, nil), + Iter: newQueryIterator(ctx, &iteratorParams{}), }, nil } @@ -69,7 +69,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if meta == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, nil, nil), + Iter: newQueryIterator(ctx, &iteratorParams{}), }, nil } @@ -86,7 +86,9 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* } return &backends.QueryResult{ - Iter: newQueryIterator(ctx, rows, nil), + Iter: newQueryIterator(ctx, &iteratorParams{ + rows: rows, + }), }, nil } @@ -278,7 +280,11 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - iter := newQueryIterator(ctx, rows, unmarshalExplain) + iter := newQueryIterator(ctx, &iteratorParams{ + rows: rows, + unmarshal: unmarshalExplain, + }) + defer iter.Close() _, queryPlan, err := iter.Next() diff --git a/internal/backends/postgresql/query_iterator.go b/internal/backends/postgresql/query_iterator.go index 0e41c55e974b..857c3787e4de 100644 --- a/internal/backends/postgresql/query_iterator.go +++ b/internal/backends/postgresql/query_iterator.go @@ -40,6 +40,12 @@ type queryIterator struct { m sync.Mutex } +// iteratorParams contains parameters for building an iterator. +type iteratorParams struct { + rows pgx.Rows + unmarshal func(b []byte) (*types.Document, error) // if set, iterator uses unmarshal to convert row to *types.Document. +} + // newQueryIterator returns a new queryIterator for the given Rows. // // Iterator's Close method closes rows. @@ -49,16 +55,16 @@ type queryIterator struct { // // Nil rows are possible and return already done iterator. // It still should be Close'd. -func newQueryIterator(ctx context.Context, rows pgx.Rows, unmarshal func(b []byte) (*types.Document, error)) types.DocumentsIterator { - if unmarshal == nil { - unmarshal = sjson.Unmarshal +func newQueryIterator(ctx context.Context, params *iteratorParams) types.DocumentsIterator { + if params.unmarshal == nil { + params.unmarshal = sjson.Unmarshal } iter := &queryIterator{ ctx: ctx, - rows: rows, + rows: params.rows, token: resource.NewToken(), - unmarshal: unmarshal, + unmarshal: params.unmarshal, } resource.Track(iter, iter.token) From d51326631a0539aac4c687b66974c0250aca1847 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 29 Sep 2023 15:50:55 +0200 Subject: [PATCH 07/21] lint --- internal/backends/postgresql/collection.go | 6 +++--- internal/backends/postgresql/helpers.go | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index d3d3877d4802..bafe54c03d12 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -58,7 +58,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if p == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, &iteratorParams{}), + Iter: newQueryIterator(ctx, new(iteratorParams)), }, nil } @@ -69,7 +69,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if meta == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, &iteratorParams{}), + Iter: newQueryIterator(ctx, new(iteratorParams)), }, nil } @@ -254,7 +254,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams } if p == nil { - return &backends.ExplainResult{}, nil + return new(backends.ExplainResult), nil } meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) diff --git a/internal/backends/postgresql/helpers.go b/internal/backends/postgresql/helpers.go index fe3824287830..3718c8d2dcf4 100644 --- a/internal/backends/postgresql/helpers.go +++ b/internal/backends/postgresql/helpers.go @@ -46,10 +46,12 @@ func convertJSON(value any) any { case map[string]any: d := types.MakeDocument(len(value)) keys := maps.Keys(value) + for _, k := range keys { v := value[k] d.Set(k, convertJSON(v)) } + return d case []any: @@ -57,6 +59,7 @@ func convertJSON(value any) any { for _, v := range value { a.Append(convertJSON(v)) } + return a case nil: From 5ca65ab3ff1fc20ae4acfa72ff3026fdfbbc7870 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 2 Oct 2023 14:33:10 +0200 Subject: [PATCH 08/21] wip --- internal/backends/postgresql/collection.go | 4 ++-- internal/backends/postgresql/query_iterator.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index bafe54c03d12..adb7dab1f788 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -58,7 +58,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if p == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, new(iteratorParams)), + Iter: newQueryIterator(ctx, nil), }, nil } @@ -69,7 +69,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* if meta == nil { return &backends.QueryResult{ - Iter: newQueryIterator(ctx, new(iteratorParams)), + Iter: newQueryIterator(ctx, nil), }, nil } diff --git a/internal/backends/postgresql/query_iterator.go b/internal/backends/postgresql/query_iterator.go index 857c3787e4de..8e0c61535425 100644 --- a/internal/backends/postgresql/query_iterator.go +++ b/internal/backends/postgresql/query_iterator.go @@ -56,6 +56,10 @@ type iteratorParams struct { // Nil rows are possible and return already done iterator. // It still should be Close'd. func newQueryIterator(ctx context.Context, params *iteratorParams) types.DocumentsIterator { + if params == nil { + params = new(iteratorParams) + } + if params.unmarshal == nil { params.unmarshal = sjson.Unmarshal } From 2d74e61088792d7c41052681f3b9a5649141dc30 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 2 Oct 2023 15:31:20 +0200 Subject: [PATCH 09/21] wip --- internal/backends/postgresql/collection.go | 127 ++++++++++++++++++++- 1 file changed, 126 insertions(+), 1 deletion(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 45acad58ed2f..948f4617ce33 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" @@ -28,6 +29,8 @@ import ( "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool" "github.com/FerretDB/FerretDB/internal/handlers/sjson" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" ) @@ -48,6 +51,129 @@ func newCollection(r *metadata.Registry, dbName, name string) backends.Collectio }) } +// prepareWhereClause adds WHERE clause with given filters to the query and returns the query and arguments. +func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []any, error) { + var filters []string + var args []any + + iter := sqlFilters.Iterator() + defer iter.Close() + + // iterate through root document + for { + rootKey, rootVal, err := iter.Next() + if err != nil { + if errors.Is(err, iterator.ErrIteratorDone) { + break + } + + return "", nil, lazyerrors.Error(err) + } + + // don't pushdown $comment, it's attached to query in handlers + if strings.HasPrefix(rootKey, "$") { + continue + } + + path, err := types.NewPathFromString(rootKey) + + var pe *types.PathError + + switch { + case err == nil: + // Handle dot notation. + // TODO https://github.com/FerretDB/FerretDB/issues/2069 + if path.Len() > 1 { + continue + } + case errors.As(err, &pe): + // ignore empty key error, otherwise return error + if pe.Code() != types.ErrPathElementEmpty { + return "", nil, lazyerrors.Error(err) + } + default: + panic("Invalid error type: PathError expected") + } + + switch v := rootVal.(type) { + case *types.Document: + iter := v.Iterator() + defer iter.Close() + + // iterate through subdocument, as it may contain operators + for { + k, v, err := iter.Next() + if err != nil { + if errors.Is(err, iterator.ErrIteratorDone) { + break + } + + return "", nil, lazyerrors.Error(err) + } + + switch k { + case "$eq": + if f, a := filterEqual(p, rootKey, v); f != "" { + filters = append(filters, f) + args = append(args, a...) + } + + case "$ne": + sql := `NOT ( ` + + // does document contain the key, + // it is necessary, as NOT won't work correctly if the key does not exist. + `_jsonb ? %[1]s AND ` + + // does the value under the key is equal to filter value + `_jsonb->%[1]s @> %[2]s AND ` + + // does the value type is equal to the filter's one + `_jsonb->'$s'->'p'->%[1]s->'t' = '"%[3]s"' )` + + switch v := v.(type) { + case *types.Document, *types.Array, types.Binary, + types.NullType, types.Regex, types.Timestamp: + // type not supported for pushdown + + case float64, bool, int32, int64: + filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) + args = append(args, rootKey, v) + + case string, types.ObjectID, time.Time: + filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) + args = append(args, rootKey, string(must.NotFail(sjson.MarshalSingleValue(v)))) + + default: + panic(fmt.Sprintf("Unexpected type of value: %v", v)) + } + + default: + // $gt and $lt + // TODO https://github.com/FerretDB/FerretDB/issues/1875 + continue + } + } + + case *types.Array, types.Binary, types.NullType, types.Regex, types.Timestamp: + // type not supported for pushdown + + case float64, string, types.ObjectID, bool, time.Time, int32, int64: + if f, a := filterEqual(p, rootKey, v); f != "" { + filters = append(filters, f) + args = append(args, a...) + } + + default: + panic(fmt.Sprintf("Unexpected type of value: %v", v)) + } + } + + var filter string + if len(filters) > 0 { + filter = ` WHERE ` + strings.Join(filters, " AND ") + } + + return filter, args, nil +} + // Query implements backends.Collection interface. func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (*backends.QueryResult, error) { p, err := c.r.DatabaseGetExisting(ctx, c.dbName) @@ -72,7 +198,6 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - // TODO https://github.com/FerretDB/FerretDB/issues/3414 q := fmt.Sprintf( `SELECT %s FROM %s`, metadata.DefaultColumn, From bdd93f1cc0125aa2a68b780d3f0315a78fbdb4b7 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 2 Oct 2023 17:14:55 +0200 Subject: [PATCH 10/21] wip --- internal/backends/postgresql/collection.go | 126 ------------ internal/backends/postgresql/query.go | 225 +++++++++++++++++++++ 2 files changed, 225 insertions(+), 126 deletions(-) create mode 100644 internal/backends/postgresql/query.go diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 948f4617ce33..f49d75441e0e 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" @@ -29,8 +28,6 @@ import ( "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool" "github.com/FerretDB/FerretDB/internal/handlers/sjson" - "github.com/FerretDB/FerretDB/internal/types" - "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" ) @@ -51,129 +48,6 @@ func newCollection(r *metadata.Registry, dbName, name string) backends.Collectio }) } -// prepareWhereClause adds WHERE clause with given filters to the query and returns the query and arguments. -func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []any, error) { - var filters []string - var args []any - - iter := sqlFilters.Iterator() - defer iter.Close() - - // iterate through root document - for { - rootKey, rootVal, err := iter.Next() - if err != nil { - if errors.Is(err, iterator.ErrIteratorDone) { - break - } - - return "", nil, lazyerrors.Error(err) - } - - // don't pushdown $comment, it's attached to query in handlers - if strings.HasPrefix(rootKey, "$") { - continue - } - - path, err := types.NewPathFromString(rootKey) - - var pe *types.PathError - - switch { - case err == nil: - // Handle dot notation. - // TODO https://github.com/FerretDB/FerretDB/issues/2069 - if path.Len() > 1 { - continue - } - case errors.As(err, &pe): - // ignore empty key error, otherwise return error - if pe.Code() != types.ErrPathElementEmpty { - return "", nil, lazyerrors.Error(err) - } - default: - panic("Invalid error type: PathError expected") - } - - switch v := rootVal.(type) { - case *types.Document: - iter := v.Iterator() - defer iter.Close() - - // iterate through subdocument, as it may contain operators - for { - k, v, err := iter.Next() - if err != nil { - if errors.Is(err, iterator.ErrIteratorDone) { - break - } - - return "", nil, lazyerrors.Error(err) - } - - switch k { - case "$eq": - if f, a := filterEqual(p, rootKey, v); f != "" { - filters = append(filters, f) - args = append(args, a...) - } - - case "$ne": - sql := `NOT ( ` + - // does document contain the key, - // it is necessary, as NOT won't work correctly if the key does not exist. - `_jsonb ? %[1]s AND ` + - // does the value under the key is equal to filter value - `_jsonb->%[1]s @> %[2]s AND ` + - // does the value type is equal to the filter's one - `_jsonb->'$s'->'p'->%[1]s->'t' = '"%[3]s"' )` - - switch v := v.(type) { - case *types.Document, *types.Array, types.Binary, - types.NullType, types.Regex, types.Timestamp: - // type not supported for pushdown - - case float64, bool, int32, int64: - filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) - args = append(args, rootKey, v) - - case string, types.ObjectID, time.Time: - filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) - args = append(args, rootKey, string(must.NotFail(sjson.MarshalSingleValue(v)))) - - default: - panic(fmt.Sprintf("Unexpected type of value: %v", v)) - } - - default: - // $gt and $lt - // TODO https://github.com/FerretDB/FerretDB/issues/1875 - continue - } - } - - case *types.Array, types.Binary, types.NullType, types.Regex, types.Timestamp: - // type not supported for pushdown - - case float64, string, types.ObjectID, bool, time.Time, int32, int64: - if f, a := filterEqual(p, rootKey, v); f != "" { - filters = append(filters, f) - args = append(args, a...) - } - - default: - panic(fmt.Sprintf("Unexpected type of value: %v", v)) - } - } - - var filter string - if len(filters) > 0 { - filter = ` WHERE ` + strings.Join(filters, " AND ") - } - - return filter, args, nil -} - // Query implements backends.Collection interface. func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (*backends.QueryResult, error) { p, err := c.r.DatabaseGetExisting(ctx, c.dbName) diff --git a/internal/backends/postgresql/query.go b/internal/backends/postgresql/query.go new file mode 100644 index 000000000000..ea88ae9a7e88 --- /dev/null +++ b/internal/backends/postgresql/query.go @@ -0,0 +1,225 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgresql + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/FerretDB/FerretDB/internal/handlers/sjson" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/iterator" + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" + "github.com/FerretDB/FerretDB/internal/util/must" +) + +// Placeholder stores the number of the relevant placeholder of the query. +type Placeholder int + +// Next increases the identifier value for the next variable in the PostgreSQL query. +func (p *Placeholder) Next() string { + *p++ + return "$" + strconv.Itoa(int(*p)) +} + +// prepareWhereClause adds WHERE clause with given filters to the query and returns the query and arguments. +func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []any, error) { + var filters []string + var args []any + + iter := sqlFilters.Iterator() + defer iter.Close() + + // iterate through root document + for { + rootKey, rootVal, err := iter.Next() + if err != nil { + if errors.Is(err, iterator.ErrIteratorDone) { + break + } + + return "", nil, lazyerrors.Error(err) + } + + // don't pushdown $comment, it's attached to query in handlers + if strings.HasPrefix(rootKey, "$") { + continue + } + + path, err := types.NewPathFromString(rootKey) + + var pe *types.PathError + + switch { + case err == nil: + // Handle dot notation. + // TODO https://github.com/FerretDB/FerretDB/issues/2069 + if path.Len() > 1 { + continue + } + case errors.As(err, &pe): + // ignore empty key error, otherwise return error + if pe.Code() != types.ErrPathElementEmpty { + return "", nil, lazyerrors.Error(err) + } + default: + panic("Invalid error type: PathError expected") + } + + switch v := rootVal.(type) { + case *types.Document: + iter := v.Iterator() + defer iter.Close() + + // iterate through subdocument, as it may contain operators + for { + k, v, err := iter.Next() + if err != nil { + if errors.Is(err, iterator.ErrIteratorDone) { + break + } + + return "", nil, lazyerrors.Error(err) + } + + switch k { + case "$eq": + if f, a := filterEqual(p, rootKey, v); f != "" { + filters = append(filters, f) + args = append(args, a...) + } + + case "$ne": + sql := `NOT ( ` + + // does document contain the key, + // it is necessary, as NOT won't work correctly if the key does not exist. + `_jsonb ? %[1]s AND ` + + // does the value under the key is equal to filter value + `_jsonb->%[1]s @> %[2]s AND ` + + // does the value type is equal to the filter's one + `_jsonb->'$s'->'p'->%[1]s->'t' = '"%[3]s"' )` + + switch v := v.(type) { + case *types.Document, *types.Array, types.Binary, + types.NullType, types.Regex, types.Timestamp: + // type not supported for pushdown + + case float64, bool, int32, int64: + filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) + args = append(args, rootKey, v) + + case string, types.ObjectID, time.Time: + filters = append(filters, fmt.Sprintf(sql, p.Next(), p.Next(), sjson.GetTypeOfValue(v))) + args = append(args, rootKey, string(must.NotFail(sjson.MarshalSingleValue(v)))) + + default: + panic(fmt.Sprintf("Unexpected type of value: %v", v)) + } + + default: + // $gt and $lt + // TODO https://github.com/FerretDB/FerretDB/issues/1875 + continue + } + } + + case *types.Array, types.Binary, types.NullType, types.Regex, types.Timestamp: + // type not supported for pushdown + + case float64, string, types.ObjectID, bool, time.Time, int32, int64: + if f, a := filterEqual(p, rootKey, v); f != "" { + filters = append(filters, f) + args = append(args, a...) + } + + default: + panic(fmt.Sprintf("Unexpected type of value: %v", v)) + } + } + + var filter string + if len(filters) > 0 { + filter = ` WHERE ` + strings.Join(filters, " AND ") + } + + return filter, args, nil +} + +// filterEqual returns the proper SQL filter with arguments that filters documents +// where the value under k is equal to v. +func filterEqual(p *Placeholder, k string, v any) (filter string, args []any) { + // Select if value under the key is equal to provided value. + sql := `_jsonb->%[1]s @> %[2]s` + + switch v := v.(type) { + case *types.Document, *types.Array, types.Binary, + types.NullType, types.Regex, types.Timestamp: + // type not supported for pushdown + + case float64: + // If value is not safe double, fetch all numbers out of safe range. + switch { + case v > types.MaxSafeDouble: + sql = `_jsonb->%[1]s > %[2]s` + v = types.MaxSafeDouble + + case v < -types.MaxSafeDouble: + sql = `_jsonb->%[1]s < %[2]s` + v = -types.MaxSafeDouble + default: + // don't change the default eq query + } + + filter = fmt.Sprintf(sql, p.Next(), p.Next()) + args = append(args, k, v) + + case string, types.ObjectID, time.Time: + // don't change the default eq query + filter = fmt.Sprintf(sql, p.Next(), p.Next()) + args = append(args, k, string(must.NotFail(sjson.MarshalSingleValue(v)))) + + case bool, int32: + // don't change the default eq query + filter = fmt.Sprintf(sql, p.Next(), p.Next()) + args = append(args, k, v) + + case int64: + maxSafeDouble := int64(types.MaxSafeDouble) + + // If value cannot be safe double, fetch all numbers out of the safe range. + switch { + case v > maxSafeDouble: + sql = `_jsonb->%[1]s > %[2]s` + v = maxSafeDouble + + case v < -maxSafeDouble: + sql = `_jsonb->%[1]s < %[2]s` + v = -maxSafeDouble + default: + // don't change the default eq query + } + + filter = fmt.Sprintf(sql, p.Next(), p.Next()) + args = append(args, k, v) + + default: + panic(fmt.Sprintf("Unexpected type of value: %v", v)) + } + + return +} From 877f2509b1937f4be2fd50bba4eb4d855f7920a6 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 2 Oct 2023 17:20:00 +0200 Subject: [PATCH 11/21] wip --- internal/backends/postgresql/collection.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index f49d75441e0e..7a720d70f1ed 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -78,7 +78,16 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), ) - rows, err := p.Query(ctx, q) + var placeholder Placeholder + + where, args, err := prepareWhereClause(&placeholder, params.Filter) + if err != nil { + return nil, lazyerrors.Error(err) + } + + q += where + + rows, err := p.Query(ctx, q, args...) if err != nil { return nil, lazyerrors.Error(err) } From 5390e76688dd42912124d930fbc2be6da61ebc72 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 3 Oct 2023 04:02:42 +0200 Subject: [PATCH 12/21] wip --- internal/backends/postgresql/collection.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 7a720d70f1ed..6a8f40e5bd2c 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -55,6 +55,10 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* return nil, lazyerrors.Error(err) } + if params == nil { + params = new(backends.QueryParams) + } + if p == nil { return &backends.QueryResult{ Iter: newQueryIterator(ctx, nil), From 8a204234dbc895d3d0d2f8a8ec2407647312fe69 Mon Sep 17 00:00:00 2001 From: Patryk Kwiatek Date: Tue, 3 Oct 2023 11:27:02 +0000 Subject: [PATCH 13/21] Update internal/backends/postgresql/collection.go Co-authored-by: Chi Fujii --- internal/backends/postgresql/collection.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index adb7dab1f788..d242651efd9b 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -275,19 +275,13 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), ) - rows, err := p.Query(ctx, q) + var b []byte + err = p.QueryRow(ctx, q).Scan(&b) if err != nil { return nil, lazyerrors.Error(err) } - iter := newQueryIterator(ctx, &iteratorParams{ - rows: rows, - unmarshal: unmarshalExplain, - }) - - defer iter.Close() - - _, queryPlan, err := iter.Next() + queryPlan, err := unmarshalExplain(b) if err != nil { return nil, lazyerrors.Error(err) } From 768b609c187ebaddecf90a15b0f37d3f41959d71 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 3 Oct 2023 13:36:40 +0200 Subject: [PATCH 14/21] wip --- internal/backends/postgresql/query_iterator.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/internal/backends/postgresql/query_iterator.go b/internal/backends/postgresql/query_iterator.go index 8e0c61535425..946ffcc92b5e 100644 --- a/internal/backends/postgresql/query_iterator.go +++ b/internal/backends/postgresql/query_iterator.go @@ -32,8 +32,6 @@ import ( type queryIterator struct { // the order of fields is weird to make the struct smaller due to alignment - unmarshal func(b []byte) (*types.Document, error) // defaults to sjson.Unmarshal - ctx context.Context rows pgx.Rows // protected by m token *resource.Token @@ -42,8 +40,7 @@ type queryIterator struct { // iteratorParams contains parameters for building an iterator. type iteratorParams struct { - rows pgx.Rows - unmarshal func(b []byte) (*types.Document, error) // if set, iterator uses unmarshal to convert row to *types.Document. + rows pgx.Rows } // newQueryIterator returns a new queryIterator for the given Rows. @@ -60,15 +57,10 @@ func newQueryIterator(ctx context.Context, params *iteratorParams) types.Documen params = new(iteratorParams) } - if params.unmarshal == nil { - params.unmarshal = sjson.Unmarshal - } - iter := &queryIterator{ - ctx: ctx, - rows: params.rows, - token: resource.NewToken(), - unmarshal: params.unmarshal, + ctx: ctx, + rows: params.rows, + token: resource.NewToken(), } resource.Track(iter, iter.token) @@ -112,7 +104,7 @@ func (iter *queryIterator) Next() (struct{}, *types.Document, error) { return unused, nil, lazyerrors.Error(err) } - doc, err := iter.unmarshal(b) + doc, err := sjson.Unmarshal(b) if err != nil { iter.close() return unused, nil, lazyerrors.Error(err) From 9ab2c51568423f35c9e3af3ae2a554b2fc3b8d99 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 3 Oct 2023 13:40:40 +0200 Subject: [PATCH 15/21] wip --- internal/backends/postgresql/collection.go | 5 ++--- internal/backends/postgresql/query_iterator.go | 13 ++----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index d242651efd9b..75ccb78ceade 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -86,9 +86,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* } return &backends.QueryResult{ - Iter: newQueryIterator(ctx, &iteratorParams{ - rows: rows, - }), + Iter: newQueryIterator(ctx, rows), }, nil } @@ -277,6 +275,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams var b []byte err = p.QueryRow(ctx, q).Scan(&b) + if err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/backends/postgresql/query_iterator.go b/internal/backends/postgresql/query_iterator.go index 946ffcc92b5e..813619cd16b5 100644 --- a/internal/backends/postgresql/query_iterator.go +++ b/internal/backends/postgresql/query_iterator.go @@ -38,11 +38,6 @@ type queryIterator struct { m sync.Mutex } -// iteratorParams contains parameters for building an iterator. -type iteratorParams struct { - rows pgx.Rows -} - // newQueryIterator returns a new queryIterator for the given Rows. // // Iterator's Close method closes rows. @@ -52,14 +47,10 @@ type iteratorParams struct { // // Nil rows are possible and return already done iterator. // It still should be Close'd. -func newQueryIterator(ctx context.Context, params *iteratorParams) types.DocumentsIterator { - if params == nil { - params = new(iteratorParams) - } - +func newQueryIterator(ctx context.Context, rows pgx.Rows) types.DocumentsIterator { iter := &queryIterator{ ctx: ctx, - rows: params.rows, + rows: rows, token: resource.NewToken(), } resource.Track(iter, iter.token) From b114287b023e4a9a8abc5cfed4fd89057c6f3f18 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 3 Oct 2023 15:06:36 +0200 Subject: [PATCH 16/21] wip --- internal/backends/postgresql/collection.go | 29 +++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index bf522c5fd96f..5f7751628dbf 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -263,8 +263,10 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } + res := new(backends.ExplainResult) + if p == nil { - return new(backends.ExplainResult), nil + return res, nil } meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) @@ -273,20 +275,29 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams } if meta == nil { - return &backends.ExplainResult{ - QueryPlanner: must.NotFail(types.NewDocument()), - }, nil + res.QueryPlanner = must.NotFail(types.NewDocument()) + return res, nil } - // TODO https://github.com/FerretDB/FerretDB/issues/3414 q := fmt.Sprintf( `EXPLAIN (VERBOSE true, FORMAT JSON) SELECT %s FROM %s`, metadata.DefaultColumn, pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), ) + var placeholder Placeholder + + where, args, err := prepareWhereClause(&placeholder, params.Filter) + if err != nil { + return nil, lazyerrors.Error(err) + } + + res.QueryPushdown = where != "" + + q += where + var b []byte - err = p.QueryRow(ctx, q).Scan(&b) + err = p.QueryRow(ctx, q, args...).Scan(&b) if err != nil { return nil, lazyerrors.Error(err) @@ -297,9 +308,9 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return nil, lazyerrors.Error(err) } - return &backends.ExplainResult{ - QueryPlanner: must.NotFail(types.NewDocument("Plan", queryPlan)), - }, nil + res.QueryPlanner = queryPlan + + return res, nil } // Stats implements backends.Collection interface. From 07596b896fa80f24f97bb64bc8b726b6bf5a037c Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 4 Oct 2023 23:47:50 +0200 Subject: [PATCH 17/21] wip --- internal/backends/postgresql/collection.go | 12 ++---------- internal/backends/postgresql/query.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index d2380ae40cec..69952c992420 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -77,11 +77,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - q := fmt.Sprintf( - `SELECT %s FROM %s`, - metadata.DefaultColumn, - pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), - ) + q := prepareSelectClause(c.dbName, meta.TableName) var placeholder Placeholder @@ -279,11 +275,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams return res, nil } - q := fmt.Sprintf( - `EXPLAIN (VERBOSE true, FORMAT JSON) SELECT %s FROM %s`, - metadata.DefaultColumn, - pgx.Identifier{c.dbName, meta.TableName}.Sanitize(), - ) + q := `EXPLAIN (VERBOSE true, FORMAT JSON) ` + prepareSelectClause(c.dbName, meta.TableName) var placeholder Placeholder diff --git a/internal/backends/postgresql/query.go b/internal/backends/postgresql/query.go index ea88ae9a7e88..28d09a3b214d 100644 --- a/internal/backends/postgresql/query.go +++ b/internal/backends/postgresql/query.go @@ -21,11 +21,13 @@ import ( "strings" "time" + "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/handlers/sjson" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" + "github.com/jackc/pgx/v5" ) // Placeholder stores the number of the relevant placeholder of the query. @@ -37,6 +39,16 @@ func (p *Placeholder) Next() string { return "$" + strconv.Itoa(int(*p)) } +// prepareSelectClause returns simple SELECT clause for provided db and table name, +// that can be used to construct the SQL query. +func prepareSelectClause(db, table string) string { + return fmt.Sprintf( + `SELECT %s FROM %s`, + metadata.DefaultColumn, + pgx.Identifier{db, table}.Sanitize(), + ) +} + // prepareWhereClause adds WHERE clause with given filters to the query and returns the query and arguments. func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []any, error) { var filters []string From 8c0fb463fedf4d69e5236ba68de01e66a656231d Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 5 Oct 2023 15:02:41 +0200 Subject: [PATCH 18/21] wip --- internal/backends/postgresql/query_test.go | 254 +++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 internal/backends/postgresql/query_test.go diff --git a/internal/backends/postgresql/query_test.go b/internal/backends/postgresql/query_test.go new file mode 100644 index 000000000000..765fe554901f --- /dev/null +++ b/internal/backends/postgresql/query_test.go @@ -0,0 +1,254 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgresql + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/must" +) + +func TestPrepareWhereClause(t *testing.T) { + t.Parallel() + objectID := types.ObjectID{0x62, 0x56, 0xc5, 0xba, 0x0b, 0xad, 0xc0, 0xff, 0xee, 0xff, 0xff, 0xff} + + // WHERE clauses occurring frequently in tests + whereContain := " WHERE _jsonb->$1 @> $2" + whereGt := " WHERE _jsonb->$1 > $2" + whereNotEq := ` WHERE NOT ( _jsonb ? $1 AND _jsonb->$1 @> $2 AND _jsonb->'$s'->'p'->$1->'t' = ` + + for name, tc := range map[string]struct { + filter *types.Document + expected string + skip string + args []any // if empty, check is disabled + }{ + "IDObjectID": { + filter: must.NotFail(types.NewDocument("_id", objectID)), + expected: whereContain, + }, + "IDString": { + filter: must.NotFail(types.NewDocument("_id", "foo")), + expected: whereContain, + }, + "IDBool": { + filter: must.NotFail(types.NewDocument("_id", "foo")), + expected: whereContain, + }, + "IDDotNotation": { + filter: must.NotFail(types.NewDocument("_id.doc", "foo")), + }, + + "DotNotation": { + filter: must.NotFail(types.NewDocument("v.doc", "foo")), + }, + "DotNotationArrayIndex": { + filter: must.NotFail(types.NewDocument("v.arr.0", "foo")), + }, + + "ImplicitString": { + filter: must.NotFail(types.NewDocument("v", "foo")), + expected: whereContain, + }, + "ImplicitEmptyString": { + filter: must.NotFail(types.NewDocument("v", "")), + expected: whereContain, + }, + "ImplicitInt32": { + filter: must.NotFail(types.NewDocument("v", int32(42))), + expected: whereContain, + }, + "ImplicitInt64": { + filter: must.NotFail(types.NewDocument("v", int64(42))), + expected: whereContain, + }, + "ImplicitFloat64": { + filter: must.NotFail(types.NewDocument("v", float64(42.13))), + expected: whereContain, + }, + "ImplicitMaxFloat64": { + filter: must.NotFail(types.NewDocument("v", math.MaxFloat64)), + expected: whereGt, + }, + "ImplicitBool": { + filter: must.NotFail(types.NewDocument("v", true)), + expected: whereContain, + }, + "ImplicitDatetime": { + filter: must.NotFail(types.NewDocument( + "v", time.Date(2021, 11, 1, 10, 18, 42, 123000000, time.UTC), + )), + expected: whereContain, + }, + "ImplicitObjectID": { + filter: must.NotFail(types.NewDocument("v", objectID)), + expected: whereContain, + }, + + "EqString": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", "foo")), + )), + args: []any{`v`, `"foo"`}, + expected: whereContain, + }, + "EqEmptyString": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", "")), + )), + expected: whereContain, + }, + "EqInt32": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", int32(42))), + )), + expected: whereContain, + }, + "EqInt64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", int64(42))), + )), + expected: whereContain, + }, + "EqFloat64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", float64(42.13))), + )), + expected: whereContain, + }, + "EqMaxFloat64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", math.MaxFloat64)), + )), + args: []any{`v`, types.MaxSafeDouble}, + expected: whereGt, + }, + "EqDoubleBigInt64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", float64(2<<61))), + )), + args: []any{`v`, types.MaxSafeDouble}, + expected: whereGt, + }, + "EqBool": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", true)), + )), + expected: whereContain, + }, + "EqDatetime": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument( + "$eq", time.Date(2021, 11, 1, 10, 18, 42, 123000000, time.UTC), + )), + )), + expected: whereContain, + }, + "EqObjectID": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$eq", objectID)), + )), + expected: whereContain, + }, + + "NeString": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", "foo")), + )), + expected: whereNotEq + `'"string"' )`, + }, + "NeEmptyString": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", "")), + )), + expected: whereNotEq + `'"string"' )`, + }, + "NeInt32": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", int32(42))), + )), + expected: whereNotEq + `'"int"' )`, + }, + "NeInt64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", int64(42))), + )), + expected: whereNotEq + `'"long"' )`, + }, + "NeFloat64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", float64(42.13))), + )), + expected: whereNotEq + `'"double"' )`, + }, + "NeMaxFloat64": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", math.MaxFloat64)), + )), + args: []any{`v`, math.MaxFloat64}, + expected: whereNotEq + `'"double"' )`, + }, + "NeBool": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", true)), + )), + expected: whereNotEq + `'"bool"' )`, + }, + "NeDatetime": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument( + "$ne", time.Date(2021, 11, 1, 10, 18, 42, 123000000, time.UTC), + )), + )), + expected: whereNotEq + `'"date"' )`, + }, + "NeObjectID": { + filter: must.NotFail(types.NewDocument( + "v", must.NotFail(types.NewDocument("$ne", objectID)), + )), + expected: whereNotEq + `'"objectId"' )`, + }, + + "Comment": { + filter: must.NotFail(types.NewDocument("$comment", "I'm comment")), + }, + } { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + if tc.skip != "" { + t.Skip(tc.skip) + } + + actual, args, err := prepareWhereClause(new(Placeholder), tc.filter) + require.NoError(t, err) + + assert.Equal(t, tc.expected, actual) + + if len(tc.args) == 0 { + return + } + + assert.Equal(t, tc.args, args) + }) + } +} From a4e473083466f65fbdf7b0ee77e052c8050d9d0d Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 5 Oct 2023 15:46:41 +0200 Subject: [PATCH 19/21] lint --- internal/backends/postgresql/query.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/backends/postgresql/query.go b/internal/backends/postgresql/query.go index 28d09a3b214d..7fa751f1f781 100644 --- a/internal/backends/postgresql/query.go +++ b/internal/backends/postgresql/query.go @@ -21,13 +21,14 @@ import ( "strings" "time" + "github.com/jackc/pgx/v5" + "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/handlers/sjson" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" - "github.com/jackc/pgx/v5" ) // Placeholder stores the number of the relevant placeholder of the query. From f4e4e6d8c7758d231c1695c0c9a9994eb0a3081c Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 5 Oct 2023 23:33:14 +0200 Subject: [PATCH 20/21] wip --- internal/backends/postgresql/query.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/internal/backends/postgresql/query.go b/internal/backends/postgresql/query.go index 7fa751f1f781..d8729972e1db 100644 --- a/internal/backends/postgresql/query.go +++ b/internal/backends/postgresql/query.go @@ -17,7 +17,6 @@ package postgresql import ( "errors" "fmt" - "strconv" "strings" "time" @@ -31,15 +30,6 @@ import ( "github.com/FerretDB/FerretDB/internal/util/must" ) -// Placeholder stores the number of the relevant placeholder of the query. -type Placeholder int - -// Next increases the identifier value for the next variable in the PostgreSQL query. -func (p *Placeholder) Next() string { - *p++ - return "$" + strconv.Itoa(int(*p)) -} - // prepareSelectClause returns simple SELECT clause for provided db and table name, // that can be used to construct the SQL query. func prepareSelectClause(db, table string) string { @@ -51,7 +41,7 @@ func prepareSelectClause(db, table string) string { } // prepareWhereClause adds WHERE clause with given filters to the query and returns the query and arguments. -func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []any, error) { +func prepareWhereClause(p *metadata.Placeholder, sqlFilters *types.Document) (string, []any, error) { var filters []string var args []any @@ -175,7 +165,7 @@ func prepareWhereClause(p *Placeholder, sqlFilters *types.Document) (string, []a // filterEqual returns the proper SQL filter with arguments that filters documents // where the value under k is equal to v. -func filterEqual(p *Placeholder, k string, v any) (filter string, args []any) { +func filterEqual(p *metadata.Placeholder, k string, v any) (filter string, args []any) { // Select if value under the key is equal to provided value. sql := `_jsonb->%[1]s @> %[2]s` From 7f23988a3940e9f857be2e8c3a1ea0ead29a22f7 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 6 Oct 2023 00:45:10 +0200 Subject: [PATCH 21/21] wip --- internal/backends/postgresql/collection.go | 4 ++-- internal/backends/postgresql/query_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 69952c992420..7f83b5b66a33 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -79,7 +79,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* q := prepareSelectClause(c.dbName, meta.TableName) - var placeholder Placeholder + var placeholder metadata.Placeholder where, args, err := prepareWhereClause(&placeholder, params.Filter) if err != nil { @@ -277,7 +277,7 @@ func (c *collection) Explain(ctx context.Context, params *backends.ExplainParams q := `EXPLAIN (VERBOSE true, FORMAT JSON) ` + prepareSelectClause(c.dbName, meta.TableName) - var placeholder Placeholder + var placeholder metadata.Placeholder where, args, err := prepareWhereClause(&placeholder, params.Filter) if err != nil { diff --git a/internal/backends/postgresql/query_test.go b/internal/backends/postgresql/query_test.go index 765fe554901f..277366bb4a7d 100644 --- a/internal/backends/postgresql/query_test.go +++ b/internal/backends/postgresql/query_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/must" ) @@ -239,7 +240,7 @@ func TestPrepareWhereClause(t *testing.T) { t.Skip(tc.skip) } - actual, args, err := prepareWhereClause(new(Placeholder), tc.filter) + actual, args, err := prepareWhereClause(new(metadata.Placeholder), tc.filter) require.NoError(t, err) assert.Equal(t, tc.expected, actual)