From 6e3df55dba8c118549f12a3cd2a968c9193aa48b Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 21 Jun 2023 00:40:37 +0200 Subject: [PATCH 01/66] wip --- internal/backends/sqlite/collection.go | 2 +- internal/backends/sqlite/metadata/registry.go | 27 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 244de22084c3..6656197e2494 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -35,7 +35,7 @@ type collection struct { name string } -// newDatabase creates a new Collection. +// newCollection creates a new Collection. func newCollection(r *metadata.Registry, dbName, name string) backends.Collection { return backends.CollectionContract(&collection{ r: r, diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index be1f4c991f4e..70ffd00bdb93 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -22,6 +22,9 @@ import ( "encoding/hex" "errors" "fmt" + "regexp" + "strings" + "unicode/utf8" "go.uber.org/zap" "modernc.org/sqlite" @@ -31,8 +34,17 @@ import ( "github.com/FerretDB/FerretDB/internal/util/lazyerrors" ) -// metadataTableName is a SQLite table name where FerretDB metadata is stored. -const metadataTableName = "_ferretdb_collections" +const ( + // Reserved prefix for database and collection names. + reservedPrefix = "_ferretdb_" + + // metadataTableName is a SQLite table name where FerretDB metadata is stored. + metadataTableName = reservedPrefix + "collections" +) + +// validateCollectionNameRe validates collection names. +// Empty collection name, names with `$` and `\x00` are not allowed. +var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") // Registry provides access to SQLite databases and collections information. type Registry struct { @@ -59,10 +71,17 @@ func (r *Registry) Close() { } // CollectionToTable converts FerretDB collection name to SQLite table name. -func (r *Registry) CollectionToTable(collectionName string) string { +func (r *Registry) CollectionToTable(collectionName string) (string, error) { + // should that be in backends.collection? + if !validateCollectionNameRe.MatchString(collectionName) || + strings.HasPrefix(collectionName, reservedPrefix) || + !utf8.ValidString(collectionName) { + return "", fmt.Errorf("Invalid collection name") + } + // TODO https://github.com/FerretDB/FerretDB/issues/2749 h := sha1.Sum([]byte(collectionName)) - return hex.EncodeToString(h[:]) + return hex.EncodeToString(h[:]), nil } // DatabaseList returns a sorted list of existing databases. From 8a2af0006634482f08147979bf1eae5384087b0a Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 01:14:52 +0200 Subject: [PATCH 02/66] wip --- internal/handlers/sqlite/msg_create.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index a7c509842d78..31e54f69ea5a 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -81,9 +81,7 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, db := h.b.Database(dbName) defer db.Close() - err = db.CreateCollection(ctx, &backends.CreateCollectionParams{ - Name: collectionName, - }) + err = createCollection(ctx, db, collectionName) switch { case err == nil: @@ -108,3 +106,11 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, lazyerrors.Error(err) } } + +func createCollection(ctx context.Context, db backends.Database, collName string) error { + err := db.CreateCollection(ctx, &backends.CreateCollectionParams{ + Name: collName, + }) + + return err +} From ae491af1ec420b170c40739eab2399bcb832ca36 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 01:38:48 +0200 Subject: [PATCH 03/66] wip --- internal/backends/sqlite/metadata/registry.go | 19 ++---------- internal/handlers/sqlite/msg_create.go | 29 +++++++++++++++---- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 70ffd00bdb93..14df9eef4ff1 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -22,9 +22,6 @@ import ( "encoding/hex" "errors" "fmt" - "regexp" - "strings" - "unicode/utf8" "go.uber.org/zap" "modernc.org/sqlite" @@ -42,10 +39,6 @@ const ( metadataTableName = reservedPrefix + "collections" ) -// validateCollectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00` are not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") - // Registry provides access to SQLite databases and collections information. type Registry struct { p *pool.Pool @@ -71,17 +64,11 @@ func (r *Registry) Close() { } // CollectionToTable converts FerretDB collection name to SQLite table name. -func (r *Registry) CollectionToTable(collectionName string) (string, error) { - // should that be in backends.collection? - if !validateCollectionNameRe.MatchString(collectionName) || - strings.HasPrefix(collectionName, reservedPrefix) || - !utf8.ValidString(collectionName) { - return "", fmt.Errorf("Invalid collection name") - } - +func (r *Registry) CollectionToTable(collectionName string) string { + // strings.HasPrefix(collectionName, reservedPrefix) || // TODO https://github.com/FerretDB/FerretDB/issues/2749 h := sha1.Sum([]byte(collectionName)) - return hex.EncodeToString(h[:]), nil + return hex.EncodeToString(h[:]) } // DatabaseList returns a sorted list of existing databases. diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 31e54f69ea5a..bbc842fd4a9e 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -16,7 +16,10 @@ package sqlite import ( "context" + "errors" "fmt" + "regexp" + "unicode/utf8" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" @@ -94,6 +97,10 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return &reply, nil + case errors.Is(err, ErrInvalidCollectionName): + msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) + return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrNamespaceExists, msg) @@ -107,10 +114,22 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } } -func createCollection(ctx context.Context, db backends.Database, collName string) error { - err := db.CreateCollection(ctx, &backends.CreateCollectionParams{ - Name: collName, - }) +// validateCollectionNameRe validates collection names. +// Empty collection name, names with `$` and `\x00` are not allowed. +var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") + +var ( + // ErrInvalidCollectionName indicates that a collection didn't pass name checks. + ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") +) + +func createCollection(ctx context.Context, db backends.Database, collectionName string) error { + if !validateCollectionNameRe.MatchString(collectionName) || + !utf8.ValidString(collectionName) { + return ErrInvalidCollectionName + } - return err + return db.CreateCollection(ctx, &backends.CreateCollectionParams{ + Name: collectionName, + }) } From 3f84105ccd8b723973f77cb27f010ea8285cc403 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 02:00:23 +0200 Subject: [PATCH 04/66] basic --- internal/backends/sqlite/metadata/registry.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 14df9eef4ff1..4f48ca6ccea9 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -66,6 +66,8 @@ func (r *Registry) Close() { // CollectionToTable converts FerretDB collection name to SQLite table name. func (r *Registry) CollectionToTable(collectionName string) string { // strings.HasPrefix(collectionName, reservedPrefix) || + // Capital letter + // TODO https://github.com/FerretDB/FerretDB/issues/2749 h := sha1.Sum([]byte(collectionName)) return hex.EncodeToString(h[:]) @@ -175,6 +177,25 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return true, nil } +func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionName string) (string, error) { + db := r.p.GetExisting(ctx, dbName) + if db == nil { + return "", nil + } + + query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) + rows, err := db.QueryContext(ctx, query, collectionName) + if err != nil { + return "", lazyerrors.Error(err) + } + defer rows.Close() + + var name string + err = rows.Scan(&name) + + return name, err +} + // CollectionDrop drops a collection in the database. // // Returned boolean value indicates whether the collection was dropped. From a5ca6cb64e89294be393c2a02b450bcd0811261f Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 14:06:28 +0200 Subject: [PATCH 05/66] remove CollectionToTable from struct --- internal/backends/sqlite/collection.go | 6 +++--- internal/backends/sqlite/metadata/registry.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 6656197e2494..68cbc6d8b9f8 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -53,7 +53,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - tableName := c.r.CollectionToTable(c.name) + tableName := metadata.CollectionToTable(c.name) query := fmt.Sprintf(`SELECT _ferretdb_sjson FROM %q`, tableName) @@ -76,7 +76,7 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) // TODO https://github.com/FerretDB/FerretDB/issues/2750 db := c.r.DatabaseGetExisting(ctx, c.dbName) - query := fmt.Sprintf(`INSERT INTO %q (_ferretdb_sjson) VALUES (?)`, c.r.CollectionToTable(c.name)) + query := fmt.Sprintf(`INSERT INTO %q (_ferretdb_sjson) VALUES (?)`, metadata.CollectionToTable(c.name)) var res backends.InsertResult @@ -117,7 +117,7 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Errorf("no database %q", c.dbName) } - tableName := c.r.CollectionToTable(c.name) + tableName := metadata.CollectionToTable(c.name) query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 4f48ca6ccea9..c3f9d82394f5 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -64,7 +64,7 @@ func (r *Registry) Close() { } // CollectionToTable converts FerretDB collection name to SQLite table name. -func (r *Registry) CollectionToTable(collectionName string) string { +func CollectionToTable(collectionName string) string { // strings.HasPrefix(collectionName, reservedPrefix) || // Capital letter @@ -153,7 +153,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - tableName := r.CollectionToTable(collectionName) + tableName := CollectionToTable(collectionName) // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 @@ -206,7 +206,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - tableName := r.CollectionToTable(collectionName) + tableName := CollectionToTable(collectionName) // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 From 88182fb290d292f22703ad6c8b7db86fb1420d96 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 14:14:12 +0200 Subject: [PATCH 06/66] Replace with collectionGet --- internal/backends/sqlite/collection.go | 17 ++++++++++++++--- internal/backends/sqlite/metadata/registry.go | 5 ++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 68cbc6d8b9f8..cd71b5ecdfc3 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -53,7 +53,10 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - tableName := metadata.CollectionToTable(c.name) + tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + if err != nil { + return nil, lazyerrors.Error(err) + } query := fmt.Sprintf(`SELECT _ferretdb_sjson FROM %q`, tableName) @@ -75,8 +78,13 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) // TODO https://github.com/FerretDB/FerretDB/issues/2750 + tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + if err != nil { + return nil, lazyerrors.Error(err) + } + db := c.r.DatabaseGetExisting(ctx, c.dbName) - query := fmt.Sprintf(`INSERT INTO %q (_ferretdb_sjson) VALUES (?)`, metadata.CollectionToTable(c.name)) + query := fmt.Sprintf(`INSERT INTO %q (_ferretdb_sjson) VALUES (?)`, tableName) var res backends.InsertResult @@ -117,7 +125,10 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Errorf("no database %q", c.dbName) } - tableName := metadata.CollectionToTable(c.name) + tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + if err != nil { + return nil, lazyerrors.Error(err) + } query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index c3f9d82394f5..da5524f5bc48 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -206,7 +206,10 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - tableName := CollectionToTable(collectionName) + tableName, err := r.CollectionGet(ctx, dbName, collectionName) + if err != nil { + return false, lazyerrors.Error(err) + } // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 From 2009ea873bc038fc6187da405b6fa463408a4fd0 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 22 Jun 2023 14:41:34 +0200 Subject: [PATCH 07/66] it works??? --- internal/backends/sqlite/metadata/registry.go | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index da5524f5bc48..9afedf7e36fe 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -190,10 +190,22 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN } defer rows.Close() - var name string - err = rows.Scan(&name) + var res []string - return name, err + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + return "", lazyerrors.Error(err) + } + + res = append(res, name) + } + + if len(res) != 1 { + return "", fmt.Errorf("res:%d", len(res)) + } + + return res[0], err } // CollectionDrop drops a collection in the database. @@ -208,7 +220,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection tableName, err := r.CollectionGet(ctx, dbName, collectionName) if err != nil { - return false, lazyerrors.Error(err) + return false, nil } // TODO use transactions From 07738c7838206d8b9c3ccb286d97e284f84e8b8e Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 23 Jun 2023 12:15:54 +0200 Subject: [PATCH 08/66] test case --- integration/basic_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/basic_test.go b/integration/basic_test.go index c2310dc10087..b1aca7b2dbb6 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -379,6 +379,9 @@ func TestCollectionName(t *testing.T) { }, altMessage: "Invalid collection name: 'TestCollectionName.\x00'", }, + "DotSurround": { + collection: ".collection..", + }, "Dot": { collection: "collection.name", }, From 004aa8e52b93647026cc545512443df3261c9992 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 23 Jun 2023 16:34:13 +0200 Subject: [PATCH 09/66] wip --- internal/backends/sqlite/metadata/registry.go | 8 +++----- internal/handlers/sqlite/msg_create.go | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 9afedf7e36fe..4996483a6556 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -63,12 +63,10 @@ func (r *Registry) Close() { r.p.Close() } -// CollectionToTable converts FerretDB collection name to SQLite table name. -func CollectionToTable(collectionName string) string { +// collectionToTable converts FerretDB collection name to SQLite table name. +func collectionToTable(collectionName string) string { // strings.HasPrefix(collectionName, reservedPrefix) || - // Capital letter - // TODO https://github.com/FerretDB/FerretDB/issues/2749 h := sha1.Sum([]byte(collectionName)) return hex.EncodeToString(h[:]) } @@ -153,7 +151,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - tableName := CollectionToTable(collectionName) + tableName := collectionToTable(collectionName) // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index bbc842fd4a9e..3c657f09dd7a 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -123,6 +123,7 @@ var ( ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") ) +// createCollection validates the collection name and creates collection. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { if !validateCollectionNameRe.MatchString(collectionName) || !utf8.ValidString(collectionName) { From 1178e6593c20140a3a316ce9450ecd8395028575 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 23 Jun 2023 16:50:46 +0200 Subject: [PATCH 10/66] prefix validation --- internal/backends/sqlite/collection.go | 3 +++ internal/backends/sqlite/database.go | 4 ++++ internal/backends/sqlite/metadata/registry.go | 21 +++++++++++++++---- internal/handlers/sqlite/msg_create.go | 13 ++++++------ internal/handlers/sqlite/msg_update.go | 5 +++++ internal/handlers/sqlite/sqlite.go | 7 +++++++ 6 files changed, 43 insertions(+), 10 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index cd71b5ecdfc3..90626220cff5 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -73,6 +73,9 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* // Insert implements backends.Collection interface. func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) (*backends.InsertResult, error) { if _, err := c.r.CollectionCreate(ctx, c.dbName, c.name); err != nil { + if errors.Is(err, metadata.ErrInvalidCollectionName) { + return nil, err + } return nil, lazyerrors.Error(err) } diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 1fc160762973..4f9070b179bd 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -16,6 +16,7 @@ package sqlite import ( "context" + "errors" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" @@ -71,6 +72,9 @@ func (db *database) ListCollections(ctx context.Context, params *backends.ListCo func (db *database) CreateCollection(ctx context.Context, params *backends.CreateCollectionParams) error { created, err := db.r.CollectionCreate(ctx, db.name, params.Name) if err != nil { + if errors.Is(err, metadata.ErrInvalidCollectionName) { + return err + } return lazyerrors.Error(err) } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 4996483a6556..ba88c7d361fd 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "errors" "fmt" + "strings" "go.uber.org/zap" "modernc.org/sqlite" @@ -39,6 +40,11 @@ const ( metadataTableName = reservedPrefix + "collections" ) +var ( + // ErrInvalidCollectionName indicates that a collection didn't pass name checks. + ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") +) + // Registry provides access to SQLite databases and collections information. type Registry struct { p *pool.Pool @@ -64,11 +70,14 @@ func (r *Registry) Close() { } // collectionToTable converts FerretDB collection name to SQLite table name. -func collectionToTable(collectionName string) string { - // strings.HasPrefix(collectionName, reservedPrefix) || +// It returns error if the name is incorrect for SQLite backend. +func collectionToTable(collectionName string) (string, error) { + if strings.HasPrefix(collectionName, reservedPrefix) { + return "", ErrInvalidCollectionName + } h := sha1.Sum([]byte(collectionName)) - return hex.EncodeToString(h[:]) + return hex.EncodeToString(h[:]), nil } // DatabaseList returns a sorted list of existing databases. @@ -145,13 +154,17 @@ func (r *Registry) CollectionList(ctx context.Context, dbName string) ([]string, // // Returned boolean value indicates whether the collection was created. // If collection already exists, (false, nil) is returned. +// If collection name doesn't pass backend level validation it returns ErrInvalidCollectionName. func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collectionName string) (bool, error) { db, err := r.DatabaseGetOrCreate(ctx, dbName) if err != nil { return false, lazyerrors.Error(err) } - tableName := collectionToTable(collectionName) + tableName, err := collectionToTable(collectionName) + if err != nil { + return false, err + } // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 3c657f09dd7a..1a30def07013 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -22,6 +22,7 @@ import ( "unicode/utf8" "github.com/FerretDB/FerretDB/internal/backends" + "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" "github.com/FerretDB/FerretDB/internal/handlers/common" "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" @@ -118,11 +119,6 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // Empty collection name, names with `$` and `\x00` are not allowed. var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") -var ( - // ErrInvalidCollectionName indicates that a collection didn't pass name checks. - ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") -) - // createCollection validates the collection name and creates collection. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { if !validateCollectionNameRe.MatchString(collectionName) || @@ -130,7 +126,12 @@ func createCollection(ctx context.Context, db backends.Database, collectionName return ErrInvalidCollectionName } - return db.CreateCollection(ctx, &backends.CreateCollectionParams{ + err := db.CreateCollection(ctx, &backends.CreateCollectionParams{ Name: collectionName, }) + if errors.Is(err, metadata.ErrInvalidCollectionName) { + return ErrInvalidCollectionName + } + + return err } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index 878dacd6feff..c96d1ca9622f 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -19,6 +19,7 @@ import ( "errors" "github.com/FerretDB/FerretDB/internal/backends" + "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" "github.com/FerretDB/FerretDB/internal/handlers/common" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" @@ -75,6 +76,10 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists) { err = nil } + if errors.Is(err, metadata.ErrInvalidCollectionName) { + return 0, 0, nil, ErrInvalidCollectionName + } + if err != nil { return 0, 0, nil, lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 67bfceb87c2c..7d0524a3e14f 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -18,6 +18,8 @@ package sqlite import ( + "fmt" + "go.uber.org/zap" "github.com/FerretDB/FerretDB/internal/backends" @@ -28,6 +30,11 @@ import ( "github.com/FerretDB/FerretDB/internal/util/state" ) +var ( + // ErrInvalidCollectionName indicates that a collection didn't pass name checks. + ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") +) + // notImplemented returns error for stub command handlers. func notImplemented(command string) error { return commonerrors.NewCommandErrorMsg( From 8c3714d368fe6972374b696156e42b38d8d21a7b Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 23 Jun 2023 17:13:14 +0200 Subject: [PATCH 11/66] tweak --- internal/backends/sqlite/metadata/registry.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index ba88c7d361fd..a22fd64c50a2 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -201,22 +201,16 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN } defer rows.Close() - var res []string - - for rows.Next() { - var name string - if err = rows.Scan(&name); err != nil { - return "", lazyerrors.Error(err) - } - - res = append(res, name) + if !rows.Next() { + return "", lazyerrors.New("no collection found") } - if len(res) != 1 { - return "", fmt.Errorf("res:%d", len(res)) + var name string + if err = rows.Scan(&name); err != nil { + return "", lazyerrors.Error(err) } - return res[0], err + return name, nil } // CollectionDrop drops a collection in the database. From 1012a4243c7daab4f6c77444f2f9434c4af841c2 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 23 Jun 2023 17:15:53 +0200 Subject: [PATCH 12/66] wip --- internal/backends/sqlite/metadata/registry.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index a22fd64c50a2..2273e1e7f921 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -76,6 +76,21 @@ func collectionToTable(collectionName string) (string, error) { return "", ErrInvalidCollectionName } + // TODO + // hash32 := fnv.New32a() + // must.NotFail(hash32.Write([]byte(name))) + + // mangled := specialCharacters.ReplaceAllString(name, "_") + + // nameSymbolsLeft := maxTableNameLength - hash32.Size()*2 - 1 + // truncateTo := len(mangled) + + // if truncateTo > nameSymbolsLeft { + // truncateTo = nameSymbolsLeft + // } + + // return mangled[:truncateTo] + "_" + hex.EncodeToString(hash32.Sum(nil)) + h := sha1.Sum([]byte(collectionName)) return hex.EncodeToString(h[:]), nil } From 7230c38ba66f018e2db8bd50a8c5336aeb1b12be Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 26 Jun 2023 12:35:07 +0200 Subject: [PATCH 13/66] fix dot for postgres --- integration/basic_test.go | 5 +++++ internal/handlers/pg/msg_create.go | 4 ++++ internal/handlers/pg/pgdb/collections.go | 8 +++++++- internal/handlers/pg/pgdb/pgdb.go | 3 +++ internal/handlers/sqlite/msg_create.go | 3 ++- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index b1aca7b2dbb6..7906d97f4dc0 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -381,6 +381,11 @@ func TestCollectionName(t *testing.T) { }, "DotSurround": { collection: ".collection..", + err: &mongo.CommandError{ + Name: "InvalidNamespace", + Code: 73, + Message: "Collection names cannot start with '.': .collection..", + }, }, "Dot": { collection: "collection.name", diff --git a/internal/handlers/pg/msg_create.go b/internal/handlers/pg/msg_create.go index 4b75a10d0d70..a0106fa529b3 100644 --- a/internal/handlers/pg/msg_create.go +++ b/internal/handlers/pg/msg_create.go @@ -105,6 +105,10 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, msg := fmt.Sprintf("Invalid collection name: '%s.%s'", db, collection) return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + case errors.Is(err, pgdb.ErrCollectionStartsWithDot): + msg := fmt.Sprintf("Collection names cannot start with '.': %s", collection) + return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + default: return lazyerrors.Error(err) } diff --git a/internal/handlers/pg/pgdb/collections.go b/internal/handlers/pg/pgdb/collections.go index d639e0df2c0d..077d9437f048 100644 --- a/internal/handlers/pg/pgdb/collections.go +++ b/internal/handlers/pg/pgdb/collections.go @@ -35,7 +35,8 @@ import ( // validateCollectionNameRe validates collection names. // Empty collection name, names with `$` and `\x00` are not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") +// TODO +var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") // Collections returns a sorted list of FerretDB collection names. // @@ -108,9 +109,14 @@ func CollectionExists(ctx context.Context, tx pgx.Tx, db, collection string) (bo // It returns possibly wrapped error: // - ErrInvalidDatabaseName - if the given database name doesn't conform to restrictions. // - ErrInvalidCollectionName - if the given collection name doesn't conform to restrictions. +// - ErrCollectionStartsWithDot - if the given collection name starts with dot. // - ErrAlreadyExist - if a FerretDB collection with the given name already exists. // - *transactionConflictError - if a PostgreSQL conflict occurs (the caller could retry the transaction). func CreateCollection(ctx context.Context, tx pgx.Tx, db, collection string) error { + if strings.HasPrefix(collection, ".") { + return ErrCollectionStartsWithDot + } + if !validateCollectionNameRe.MatchString(collection) || strings.HasPrefix(collection, reservedPrefix) || !utf8.ValidString(collection) { diff --git a/internal/handlers/pg/pgdb/pgdb.go b/internal/handlers/pg/pgdb/pgdb.go index e2e69a2e3e07..e2f6c318b6e6 100644 --- a/internal/handlers/pg/pgdb/pgdb.go +++ b/internal/handlers/pg/pgdb/pgdb.go @@ -45,6 +45,9 @@ var ( // ErrInvalidCollectionName indicates that a collection didn't pass name checks. ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") + // ErrCollectionStartsWithDot indicates that collection name starts with dot, but it shouldn't. + ErrCollectionStartsWithDot = fmt.Errorf("Collection names cannot start with '.'") + // ErrInvalidDatabaseName indicates that a database didn't pass name checks. ErrInvalidDatabaseName = fmt.Errorf("invalid FerretDB database name") diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 1a30def07013..bad1efc997f1 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -117,7 +117,8 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // validateCollectionNameRe validates collection names. // Empty collection name, names with `$` and `\x00` are not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") +// TODO +var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") // createCollection validates the collection name and creates collection. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { From 176cb059183de06935487e755ae8b0766d848d09 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 26 Jun 2023 12:50:00 +0200 Subject: [PATCH 14/66] fix dot for sqlite --- internal/handlers/sqlite/msg_create.go | 9 +++++++++ internal/handlers/sqlite/sqlite.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index bad1efc997f1..8b2753494b97 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "regexp" + "strings" "unicode/utf8" "github.com/FerretDB/FerretDB/internal/backends" @@ -102,6 +103,10 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + case errors.Is(err, ErrCollectionStartsWithDot): + msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) + return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrNamespaceExists, msg) @@ -122,6 +127,10 @@ var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") // createCollection validates the collection name and creates collection. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { + if strings.HasPrefix(collectionName, ".") { + return ErrCollectionStartsWithDot + } + if !validateCollectionNameRe.MatchString(collectionName) || !utf8.ValidString(collectionName) { return ErrInvalidCollectionName diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 7d0524a3e14f..2500ee300e82 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -33,6 +33,9 @@ import ( var ( // ErrInvalidCollectionName indicates that a collection didn't pass name checks. ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") + + // ErrCollectionStartsWithDot indicates that collection name starts with dot, but it shouldn't. + ErrCollectionStartsWithDot = fmt.Errorf("Collection names cannot start with '.'") ) // notImplemented returns error for stub command handlers. From 2fbbd645e114ead3ac9a666ad67837fb2d3ce14f Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 26 Jun 2023 21:30:24 +0200 Subject: [PATCH 15/66] Update sqlite comment --- internal/handlers/sqlite/msg_create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 8b2753494b97..3887ebda0e7e 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -122,7 +122,7 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // validateCollectionNameRe validates collection names. // Empty collection name, names with `$` and `\x00` are not allowed. -// TODO +// Collection names that start with `.` are also not allowed. var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") // createCollection validates the collection name and creates collection. From dac0fa96a0a93f67f971bcef3efcf3fdf0733ea8 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 26 Jun 2023 23:39:20 +0200 Subject: [PATCH 16/66] wip --- integration/basic_test.go | 3 ++ internal/backends/sqlite/metadata/registry.go | 38 ++++++++++++------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 7906d97f4dc0..06cca877177f 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -405,6 +405,9 @@ func TestCollectionName(t *testing.T) { "Capital": { collection: "A", }, + "sqlite": { + collection: "sqlite", + }, } for name, tc := range cases { diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 2273e1e7f921..128923a09e48 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -17,11 +17,11 @@ package metadata import ( "context" - "crypto/sha1" "database/sql" "encoding/hex" "errors" "fmt" + "hash/fnv" "strings" "go.uber.org/zap" @@ -30,12 +30,16 @@ import ( "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata/pool" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" + "github.com/FerretDB/FerretDB/internal/util/must" ) const ( // Reserved prefix for database and collection names. reservedPrefix = "_ferretdb_" + // Reserved prefix for SQLite table name. + reservedTablePrefix = "sqlite" + // metadataTableName is a SQLite table name where FerretDB metadata is stored. metadataTableName = reservedPrefix + "collections" ) @@ -76,23 +80,31 @@ func collectionToTable(collectionName string) (string, error) { return "", ErrInvalidCollectionName } - // TODO - // hash32 := fnv.New32a() - // must.NotFail(hash32.Write([]byte(name))) + hash32 := fnv.New32a() + must.NotFail(hash32.Write([]byte(collectionName))) + + // SQLite table cannot start with _sqlite prefix + if strings.HasPrefix(collectionName, reservedTablePrefix) { + collectionName = "_" + collectionName + } - // mangled := specialCharacters.ReplaceAllString(name, "_") + // TODO https://www.tutlane.com/tutorial/sqlite/sqlite-syntax + // - case insensitive + // - must begin with letter or underscore character which is followed by any alphanuymeric character or underscore. Other characters are invalid + // - // nameSymbolsLeft := maxTableNameLength - hash32.Size()*2 - 1 - // truncateTo := len(mangled) + //nameSymbolsLeft := maxTableNameLength - hash32.Size()*2 - 1 + //truncateTo := len(mangled) - // if truncateTo > nameSymbolsLeft { - // truncateTo = nameSymbolsLeft - // } + //if truncateTo > nameSymbolsLeft { + // truncateTo = nameSymbolsLeft + //} - // return mangled[:truncateTo] + "_" + hex.EncodeToString(hash32.Sum(nil)) + //return mangled[:truncateTo] + "_" + hex.EncodeToString(hash32.Sum(nil)) + return collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)), nil - h := sha1.Sum([]byte(collectionName)) - return hex.EncodeToString(h[:]), nil + //h := sha1.Sum([]byte(collectionName)) + //return hex.EncodeToString(h[:]), nil } // DatabaseList returns a sorted list of existing databases. From f97a473acd073bb622f211b4e88cdfdfff0834b4 Mon Sep 17 00:00:00 2001 From: noisersup Date: Mon, 26 Jun 2023 23:58:04 +0200 Subject: [PATCH 17/66] add sqlite validation --- internal/backends/sqlite/metadata/registry.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 128923a09e48..57672b4ac9b8 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -84,27 +84,12 @@ func collectionToTable(collectionName string) (string, error) { must.NotFail(hash32.Write([]byte(collectionName))) // SQLite table cannot start with _sqlite prefix - if strings.HasPrefix(collectionName, reservedTablePrefix) { + if strings.HasPrefix(strings.ToLower(collectionName), reservedTablePrefix) { collectionName = "_" + collectionName } + // TODO case insensitive - // TODO https://www.tutlane.com/tutorial/sqlite/sqlite-syntax - // - case insensitive - // - must begin with letter or underscore character which is followed by any alphanuymeric character or underscore. Other characters are invalid - // - - //nameSymbolsLeft := maxTableNameLength - hash32.Size()*2 - 1 - //truncateTo := len(mangled) - - //if truncateTo > nameSymbolsLeft { - // truncateTo = nameSymbolsLeft - //} - - //return mangled[:truncateTo] + "_" + hex.EncodeToString(hash32.Sum(nil)) return collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)), nil - - //h := sha1.Sum([]byte(collectionName)) - //return hex.EncodeToString(h[:]), nil } // DatabaseList returns a sorted list of existing databases. From c179a5095dfb4c5e64f09e0bd77d8e12420561fc Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 08:19:14 +0200 Subject: [PATCH 18/66] add comment + lint --- internal/backends/sqlite/metadata/registry.go | 4 +++- internal/handlers/sqlite/msg_update.go | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 57672b4ac9b8..eeadebe4d5fd 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -40,7 +40,7 @@ const ( // Reserved prefix for SQLite table name. reservedTablePrefix = "sqlite" - // metadataTableName is a SQLite table name where FerretDB metadata is stored. + // SQLite table name where FerretDB metadata is stored. metadataTableName = reservedPrefix + "collections" ) @@ -200,6 +200,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return true, nil } +// CollectionGet returns table name associated with provided collection. func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionName string) (string, error) { db := r.p.GetExisting(ctx, dbName) if db == nil { @@ -208,6 +209,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) rows, err := db.QueryContext(ctx, query, collectionName) + if err != nil { return "", lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index d240a88d101b..9af7637a7f18 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -76,6 +76,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists) { err = nil } + if errors.Is(err, metadata.ErrInvalidCollectionName) { return 0, 0, nil, ErrInvalidCollectionName } From 4bfe05b91ecc2b5a42a050b8a0fa8bb690f1ed11 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 09:04:25 +0200 Subject: [PATCH 19/66] fmt --- internal/backends/sqlite/metadata/registry.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index eeadebe4d5fd..80ce8597fa13 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -44,10 +44,8 @@ const ( metadataTableName = reservedPrefix + "collections" ) -var ( - // ErrInvalidCollectionName indicates that a collection didn't pass name checks. - ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") -) +// ErrInvalidCollectionName indicates that a collection didn't pass name checks. +var ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") // Registry provides access to SQLite databases and collections information. type Registry struct { @@ -208,8 +206,8 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN } query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) - rows, err := db.QueryContext(ctx, query, collectionName) + rows, err := db.QueryContext(ctx, query, collectionName) if err != nil { return "", lazyerrors.Error(err) } From 4d9828160755d55c7cf7e5ba8f9776d395ba4f3e Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 14:11:44 +0200 Subject: [PATCH 20/66] wip --- integration/basic_test.go | 2 +- internal/backends/sqlite/metadata/registry.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 06cca877177f..6686e84cfa50 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -406,7 +406,7 @@ func TestCollectionName(t *testing.T) { collection: "A", }, "sqlite": { - collection: "sqlite", + collection: "sqlite_", }, } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 80ce8597fa13..4a9d79b1ad7f 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -38,6 +38,7 @@ const ( reservedPrefix = "_ferretdb_" // Reserved prefix for SQLite table name. + // https://www.sqlite.org/lang_createtable.html reservedTablePrefix = "sqlite" // SQLite table name where FerretDB metadata is stored. @@ -72,7 +73,7 @@ func (r *Registry) Close() { } // collectionToTable converts FerretDB collection name to SQLite table name. -// It returns error if the name is incorrect for SQLite backend. +// It returns ErrInvalidCollectionName error if the name is incorrect for SQLite backend. func collectionToTable(collectionName string) (string, error) { if strings.HasPrefix(collectionName, reservedPrefix) { return "", ErrInvalidCollectionName @@ -199,6 +200,8 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti } // CollectionGet returns table name associated with provided collection. +// +// If database does not exist, no error is returned. func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionName string) (string, error) { db := r.p.GetExisting(ctx, dbName) if db == nil { @@ -215,6 +218,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN if !rows.Next() { return "", lazyerrors.New("no collection found") + // TODO return ErrDontExist or smth } var name string @@ -237,6 +241,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection tableName, err := r.CollectionGet(ctx, dbName, collectionName) if err != nil { + // TODO return nil for nonexistent collection but lazyerror for anything else return false, nil } From 0332333afa15387636565a6a47b6383a350e9442 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 15:07:29 +0200 Subject: [PATCH 21/66] wip --- internal/handlers/pg/pgdb/collections.go | 3 ++- internal/handlers/sqlite/msg_create.go | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/handlers/pg/pgdb/collections.go b/internal/handlers/pg/pgdb/collections.go index dce2d369e6b3..7a8d6f4aba69 100644 --- a/internal/handlers/pg/pgdb/collections.go +++ b/internal/handlers/pg/pgdb/collections.go @@ -34,7 +34,8 @@ import ( ) // validateCollectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00` are not allowed. +// Empty collection name, names with `$` and `\x00`, +// or exceeding the 255 bytes limit are not allowed. // Collection names that start with `.` are also not allowed. var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 3887ebda0e7e..095bc89f8911 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -101,19 +101,19 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, case errors.Is(err, ErrInvalidCollectionName): msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") case errors.Is(err, ErrCollectionStartsWithDot): msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) - return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrNamespaceExists, msg) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrNamespaceExists, msg, "create") case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): msg := fmt.Sprintf("Invalid namespace: %s.%s", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") default: return nil, lazyerrors.Error(err) @@ -121,11 +121,14 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } // validateCollectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00` are not allowed. +// Empty collection name, names with `$` and `\x00`, +// or exceeding the 255 bytes limit are not allowed. // Collection names that start with `.` are also not allowed. var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") // createCollection validates the collection name and creates collection. +// It returns ErrInvalidCollectionName when the collection name is invalid. +// It returns ErrCollectionStartsWithDot if collection starts with a dot. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { if strings.HasPrefix(collectionName, ".") { return ErrCollectionStartsWithDot From 257e8660945ca8095de7a01c689c432558717559 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 15:32:33 +0200 Subject: [PATCH 22/66] make all table names lowercase --- internal/backends/sqlite/metadata/registry.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 4a9d79b1ad7f..e5283e4f7f4b 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -82,11 +82,12 @@ func collectionToTable(collectionName string) (string, error) { hash32 := fnv.New32a() must.NotFail(hash32.Write([]byte(collectionName))) + collectionName = strings.ToLower(collectionName) + // SQLite table cannot start with _sqlite prefix - if strings.HasPrefix(strings.ToLower(collectionName), reservedTablePrefix) { + if strings.HasPrefix(collectionName, reservedTablePrefix) { collectionName = "_" + collectionName } - // TODO case insensitive return collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)), nil } From 7505d9a576404031d74e1f7c6f61ae87c90666e5 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 16:15:40 +0200 Subject: [PATCH 23/66] proper error handling --- internal/backends/sqlite/metadata/registry.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index e5283e4f7f4b..69abfa7344a0 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -45,8 +45,13 @@ const ( metadataTableName = reservedPrefix + "collections" ) -// ErrInvalidCollectionName indicates that a collection didn't pass name checks. -var ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") +var ( + // ErrInvalidCollectionName indicates that a collection didn't pass name checks. + ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") + + // ErrCollectionDoesNotExist indicates that collection does not exist. + ErrCollectionDoesNotExist = fmt.Errorf("collection does not exist") +) // Registry provides access to SQLite databases and collections information. type Registry struct { @@ -218,8 +223,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN defer rows.Close() if !rows.Next() { - return "", lazyerrors.New("no collection found") - // TODO return ErrDontExist or smth + return "", ErrCollectionDoesNotExist } var name string @@ -241,11 +245,14 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection } tableName, err := r.CollectionGet(ctx, dbName, collectionName) - if err != nil { - // TODO return nil for nonexistent collection but lazyerror for anything else + if errors.Is(err, ErrCollectionDoesNotExist) { return false, nil } + if err != nil { + return false, lazyerrors.Error(err) + } + // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 From f05bf627e5b8b3fdda33ca1534a5d1dcb62c2612 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 20:03:57 +0200 Subject: [PATCH 24/66] lot of changes --- internal/backends/error.go | 1 + internal/backends/sqlite/collection.go | 3 +- internal/backends/sqlite/database.go | 4 +- internal/backends/sqlite/metadata/registry.go | 15 ++------ internal/handlers/sqlite/msg_create.go | 37 ++++++++----------- internal/handlers/sqlite/msg_update.go | 6 +-- internal/handlers/sqlite/sqlite.go | 10 ----- 7 files changed, 28 insertions(+), 48 deletions(-) diff --git a/internal/backends/error.go b/internal/backends/error.go index 096a9c66eb6e..92154238f869 100644 --- a/internal/backends/error.go +++ b/internal/backends/error.go @@ -36,6 +36,7 @@ const ( ErrorCodeCollectionDoesNotExist ErrorCodeCollectionAlreadyExists ErrorCodeCollectionNameIsInvalid + ErrorCodeCollectionStartsWithDot ) // Error represents a backend error returned by all Backend, Database and Collection methods. diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 90626220cff5..e1e237c13e53 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -73,7 +73,8 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* // Insert implements backends.Collection interface. func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) (*backends.InsertResult, error) { if _, err := c.r.CollectionCreate(ctx, c.dbName, c.name); err != nil { - if errors.Is(err, metadata.ErrInvalidCollectionName) { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + //TODO return nil, err } return nil, lazyerrors.Error(err) diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 4f9070b179bd..09bafbee771d 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -16,7 +16,6 @@ package sqlite import ( "context" - "errors" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" @@ -72,7 +71,8 @@ func (db *database) ListCollections(ctx context.Context, params *backends.ListCo func (db *database) CreateCollection(ctx context.Context, params *backends.CreateCollectionParams) error { created, err := db.r.CollectionCreate(ctx, db.name, params.Name) if err != nil { - if errors.Is(err, metadata.ErrInvalidCollectionName) { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + //TODO return err } return lazyerrors.Error(err) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 69abfa7344a0..53658dec6113 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -28,6 +28,7 @@ import ( "modernc.org/sqlite" sqlitelib "modernc.org/sqlite/lib" + "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata/pool" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -45,14 +46,6 @@ const ( metadataTableName = reservedPrefix + "collections" ) -var ( - // ErrInvalidCollectionName indicates that a collection didn't pass name checks. - ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") - - // ErrCollectionDoesNotExist indicates that collection does not exist. - ErrCollectionDoesNotExist = fmt.Errorf("collection does not exist") -) - // Registry provides access to SQLite databases and collections information. type Registry struct { p *pool.Pool @@ -81,7 +74,7 @@ func (r *Registry) Close() { // It returns ErrInvalidCollectionName error if the name is incorrect for SQLite backend. func collectionToTable(collectionName string) (string, error) { if strings.HasPrefix(collectionName, reservedPrefix) { - return "", ErrInvalidCollectionName + return "", backends.NewError(backends.ErrorCodeCollectionNameIsInvalid, nil) } hash32 := fnv.New32a() @@ -223,7 +216,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN defer rows.Close() if !rows.Next() { - return "", ErrCollectionDoesNotExist + return "", backends.NewError(backends.ErrorCodeCollectionDoesNotExist, nil) } var name string @@ -245,7 +238,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection } tableName, err := r.CollectionGet(ctx, dbName, collectionName) - if errors.Is(err, ErrCollectionDoesNotExist) { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { return false, nil } diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 095bc89f8911..da9f2b766d7e 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -16,14 +16,12 @@ package sqlite import ( "context" - "errors" "fmt" "regexp" "strings" "unicode/utf8" "github.com/FerretDB/FerretDB/internal/backends" - "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" "github.com/FerretDB/FerretDB/internal/handlers/common" "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" @@ -99,11 +97,11 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return &reply, nil - case errors.Is(err, ErrInvalidCollectionName): + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - case errors.Is(err, ErrCollectionStartsWithDot): + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionStartsWithDot): msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") @@ -111,40 +109,37 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrNamespaceExists, msg, "create") - case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): - msg := fmt.Sprintf("Invalid namespace: %s.%s", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - default: return nil, lazyerrors.Error(err) } } -// validateCollectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00`, -// or exceeding the 255 bytes limit are not allowed. -// Collection names that start with `.` are also not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") - // createCollection validates the collection name and creates collection. // It returns ErrInvalidCollectionName when the collection name is invalid. // It returns ErrCollectionStartsWithDot if collection starts with a dot. func createCollection(ctx context.Context, db backends.Database, collectionName string) error { + // validateCollectionNameRe validates collection names. + // Empty collection name, names with `$` and `\x00`, + // or exceeding the 255 bytes limit are not allowed. + // Collection names that start with `.` are also not allowed. + validateCollectionNameRe := regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") + if strings.HasPrefix(collectionName, ".") { - return ErrCollectionStartsWithDot + return backends.NewError(backends.ErrorCodeCollectionStartsWithDot, nil) } if !validateCollectionNameRe.MatchString(collectionName) || !utf8.ValidString(collectionName) { - return ErrInvalidCollectionName + return backends.NewError(backends.ErrorCodeCollectionNameIsInvalid, nil) } - err := db.CreateCollection(ctx, &backends.CreateCollectionParams{ + return db.CreateCollection(ctx, &backends.CreateCollectionParams{ Name: collectionName, }) - if errors.Is(err, metadata.ErrInvalidCollectionName) { - return ErrInvalidCollectionName - } - return err + //if errors.Is(err, metadata.ErrInvalidCollectionName) { + // return ErrInvalidCollectionName + //} + + //return err } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index 9af7637a7f18..368b44517890 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -19,7 +19,6 @@ import ( "errors" "github.com/FerretDB/FerretDB/internal/backends" - "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" "github.com/FerretDB/FerretDB/internal/handlers/common" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" @@ -77,8 +76,9 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara err = nil } - if errors.Is(err, metadata.ErrInvalidCollectionName) { - return 0, 0, nil, ErrInvalidCollectionName + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + //TODO: check if its needed + return 0, 0, nil, err } if err != nil { diff --git a/internal/handlers/sqlite/sqlite.go b/internal/handlers/sqlite/sqlite.go index 01e3381dd824..b01390ede4a4 100644 --- a/internal/handlers/sqlite/sqlite.go +++ b/internal/handlers/sqlite/sqlite.go @@ -18,8 +18,6 @@ package sqlite import ( - "fmt" - "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -32,14 +30,6 @@ import ( "github.com/FerretDB/FerretDB/internal/util/state" ) -var ( - // ErrInvalidCollectionName indicates that a collection didn't pass name checks. - ErrInvalidCollectionName = fmt.Errorf("invalid FerretDB collection name") - - // ErrCollectionStartsWithDot indicates that collection name starts with dot, but it shouldn't. - ErrCollectionStartsWithDot = fmt.Errorf("Collection names cannot start with '.'") -) - // notImplemented returns error for stub command handlers. func notImplemented(command string) error { return commonerrors.NewCommandErrorMsg( From 1ef2680504910f4d4a6443251b3e063cf42d1854 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 20:43:04 +0200 Subject: [PATCH 25/66] wip --- internal/backends/errorcode_string.go | 5 +++-- internal/backends/sqlite/collection.go | 2 +- internal/backends/sqlite/database.go | 2 +- internal/handlers/sqlite/msg_create.go | 2 +- internal/handlers/sqlite/msg_update.go | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/backends/errorcode_string.go b/internal/backends/errorcode_string.go index e04ad00dd5a1..9bc9993516dd 100644 --- a/internal/backends/errorcode_string.go +++ b/internal/backends/errorcode_string.go @@ -12,11 +12,12 @@ func _() { _ = x[ErrorCodeCollectionDoesNotExist-2] _ = x[ErrorCodeCollectionAlreadyExists-3] _ = x[ErrorCodeCollectionNameIsInvalid-4] + _ = x[ErrorCodeCollectionStartsWithDot-5] } -const _ErrorCode_name = "ErrorCodeDatabaseDoesNotExistErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExistsErrorCodeCollectionNameIsInvalid" +const _ErrorCode_name = "ErrorCodeDatabaseDoesNotExistErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExistsErrorCodeCollectionNameIsInvalidErrorCodeCollectionStartsWithDot" -var _ErrorCode_index = [...]uint8{0, 29, 60, 92, 124} +var _ErrorCode_index = [...]uint8{0, 29, 60, 92, 124, 156} func (i ErrorCode) String() string { i -= 1 diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index e1e237c13e53..1ecd6ea94f4d 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -74,7 +74,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) (*backends.InsertResult, error) { if _, err := c.r.CollectionCreate(ctx, c.dbName, c.name); err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - //TODO + // TODO return nil, err } return nil, lazyerrors.Error(err) diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 09bafbee771d..aae8ec421193 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -72,7 +72,7 @@ func (db *database) CreateCollection(ctx context.Context, params *backends.Creat created, err := db.r.CollectionCreate(ctx, db.name, params.Name) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - //TODO + // TODO return err } return lazyerrors.Error(err) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index da9f2b766d7e..aac850aca277 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -141,5 +141,5 @@ func createCollection(ctx context.Context, db backends.Database, collectionName // return ErrInvalidCollectionName //} - //return err + // return err } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index 368b44517890..be90fa5c0b47 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -77,7 +77,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara } if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - //TODO: check if its needed + // TODO: check if its needed return 0, 0, nil, err } From a16bb207970b006289dab2142e80b07450d8148d Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 20:58:08 +0200 Subject: [PATCH 26/66] fix --- integration/basic_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 6686e84cfa50..8be3185e9318 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -405,7 +405,7 @@ func TestCollectionName(t *testing.T) { "Capital": { collection: "A", }, - "sqlite": { + "Sqlite": { collection: "sqlite_", }, } @@ -417,7 +417,8 @@ func TestCollectionName(t *testing.T) { t.Skip(tc.skip) } - t.Parallel() + // TODO https://github.com/FerretDB/FerretDB/issues/2923 + // t.Parallel() err := collection.Database().CreateCollection(ctx, tc.collection) if tc.err != nil { From 12a80e49ead36ee6692689caee2e516dac8af83d Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 21:28:33 +0200 Subject: [PATCH 27/66] wip --- internal/backends/sqlite/database.go | 4 ---- internal/backends/sqlite/metadata/registry.go | 5 ++++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index aae8ec421193..1fc160762973 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -71,10 +71,6 @@ func (db *database) ListCollections(ctx context.Context, params *backends.ListCo func (db *database) CreateCollection(ctx context.Context, params *backends.CreateCollectionParams) error { created, err := db.r.CollectionCreate(ctx, db.name, params.Name) if err != nil { - if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - // TODO - return err - } return lazyerrors.Error(err) } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 53658dec6113..32d51f92932a 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -216,7 +216,10 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN defer rows.Close() if !rows.Next() { - return "", backends.NewError(backends.ErrorCodeCollectionDoesNotExist, nil) + return "", backends.NewError( + backends.ErrorCodeCollectionDoesNotExist, + fmt.Errorf("Collection %q does not exist", collectionName), + ) } var name string From 2db636dc47989b9c66397ba040a77e20c315ddca Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 21:37:09 +0200 Subject: [PATCH 28/66] wip --- internal/backends/sqlite/collection.go | 4 ---- internal/handlers/sqlite/msg_create.go | 5 ----- internal/handlers/sqlite/msg_update.go | 5 ----- 3 files changed, 14 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 1ecd6ea94f4d..cd71b5ecdfc3 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -73,10 +73,6 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* // Insert implements backends.Collection interface. func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) (*backends.InsertResult, error) { if _, err := c.r.CollectionCreate(ctx, c.dbName, c.name); err != nil { - if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - // TODO - return nil, err - } return nil, lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index aac850aca277..188cf4d628cb 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -137,9 +137,4 @@ func createCollection(ctx context.Context, db backends.Database, collectionName Name: collectionName, }) - //if errors.Is(err, metadata.ErrInvalidCollectionName) { - // return ErrInvalidCollectionName - //} - - // return err } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index be90fa5c0b47..ec3240901c06 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -76,11 +76,6 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara err = nil } - if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - // TODO: check if its needed - return 0, 0, nil, err - } - if err != nil { return 0, 0, nil, lazyerrors.Error(err) } From ef77cfe29145ef212c89bc70d7acc47707553529 Mon Sep 17 00:00:00 2001 From: noisersup Date: Tue, 27 Jun 2023 21:42:22 +0200 Subject: [PATCH 29/66] wip --- internal/handlers/sqlite/msg_create.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 188cf4d628cb..5ffaa850f815 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -136,5 +136,4 @@ func createCollection(ctx context.Context, db backends.Database, collectionName return db.CreateCollection(ctx, &backends.CreateCollectionParams{ Name: collectionName, }) - } From 494dcd7a8e04c11ac00369fe1dd88d5698519dde Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 28 Jun 2023 12:54:25 +0200 Subject: [PATCH 30/66] pair programming --- internal/backends/database.go | 2 +- internal/backends/error.go | 2 - internal/backends/errorcode_string.go | 6 +- internal/backends/sqlite/metadata/registry.go | 4 -- internal/handlers/sqlite/msg_create.go | 61 +++++++++---------- 5 files changed, 31 insertions(+), 44 deletions(-) diff --git a/internal/backends/database.go b/internal/backends/database.go index 4efd9e33829f..70f277abf2bd 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -114,7 +114,7 @@ type CreateCollectionParams struct { // Database may or may not exist; it should be created automatically if needed. func (dbc *databaseContract) CreateCollection(ctx context.Context, params *CreateCollectionParams) (err error) { defer observability.FuncCall(ctx)() - defer checkError(err, ErrorCodeCollectionAlreadyExists, ErrorCodeCollectionNameIsInvalid) + defer checkError(err, ErrorCodeCollectionAlreadyExists) err = dbc.db.CreateCollection(ctx, params) return diff --git a/internal/backends/error.go b/internal/backends/error.go index 92154238f869..8210267b0ab4 100644 --- a/internal/backends/error.go +++ b/internal/backends/error.go @@ -35,8 +35,6 @@ const ( ErrorCodeCollectionDoesNotExist ErrorCodeCollectionAlreadyExists - ErrorCodeCollectionNameIsInvalid - ErrorCodeCollectionStartsWithDot ) // Error represents a backend error returned by all Backend, Database and Collection methods. diff --git a/internal/backends/errorcode_string.go b/internal/backends/errorcode_string.go index 9bc9993516dd..2444c28eb6c3 100644 --- a/internal/backends/errorcode_string.go +++ b/internal/backends/errorcode_string.go @@ -11,13 +11,11 @@ func _() { _ = x[ErrorCodeDatabaseDoesNotExist-1] _ = x[ErrorCodeCollectionDoesNotExist-2] _ = x[ErrorCodeCollectionAlreadyExists-3] - _ = x[ErrorCodeCollectionNameIsInvalid-4] - _ = x[ErrorCodeCollectionStartsWithDot-5] } -const _ErrorCode_name = "ErrorCodeDatabaseDoesNotExistErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExistsErrorCodeCollectionNameIsInvalidErrorCodeCollectionStartsWithDot" +const _ErrorCode_name = "ErrorCodeDatabaseDoesNotExistErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExists" -var _ErrorCode_index = [...]uint8{0, 29, 60, 92, 124, 156} +var _ErrorCode_index = [...]uint8{0, 29, 60, 92} func (i ErrorCode) String() string { i -= 1 diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 32d51f92932a..1b7b8d9cd2fc 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -73,10 +73,6 @@ func (r *Registry) Close() { // collectionToTable converts FerretDB collection name to SQLite table name. // It returns ErrInvalidCollectionName error if the name is incorrect for SQLite backend. func collectionToTable(collectionName string) (string, error) { - if strings.HasPrefix(collectionName, reservedPrefix) { - return "", backends.NewError(backends.ErrorCodeCollectionNameIsInvalid, nil) - } - hash32 := fnv.New32a() must.NotFail(hash32.Write([]byte(collectionName))) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 5ffaa850f815..11650638ed11 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -30,6 +30,15 @@ import ( "github.com/FerretDB/FerretDB/internal/wire" ) +// Reserved prefix for database and collection names. +const reservedPrefix = "_ferretdb_" + +// validateCollectionNameRe validates collection names. +// Empty collection name, names with `$` and `\x00`, +// or exceeding the 255 bytes limit are not allowed. +// Collection names that start with `.` are also not allowed. +var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") + // MsgCreate implements HandlerInterface. func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, error) { document, err := msg.Document() @@ -81,10 +90,28 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, err } + if strings.HasPrefix(collectionName, ".") { + msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + } + + if !validateCollectionNameRe.MatchString(collectionName) || + !utf8.ValidString(collectionName) { + msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + } + + if strings.HasPrefix(collectionName, reservedPrefix) { + msg := fmt.Sprintf("Invalid collection name (%q is a reserved prefix): '%s.%s'", reservedPrefix, dbName, collectionName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + } + db := h.b.Database(dbName) defer db.Close() - err = createCollection(ctx, db, collectionName) + err = db.CreateCollection(ctx, &backends.CreateCollectionParams{ + Name: collectionName, + }) switch { case err == nil: @@ -97,14 +124,6 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return &reply, nil - case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): - msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - - case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionStartsWithDot): - msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrNamespaceExists, msg, "create") @@ -113,27 +132,3 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, lazyerrors.Error(err) } } - -// createCollection validates the collection name and creates collection. -// It returns ErrInvalidCollectionName when the collection name is invalid. -// It returns ErrCollectionStartsWithDot if collection starts with a dot. -func createCollection(ctx context.Context, db backends.Database, collectionName string) error { - // validateCollectionNameRe validates collection names. - // Empty collection name, names with `$` and `\x00`, - // or exceeding the 255 bytes limit are not allowed. - // Collection names that start with `.` are also not allowed. - validateCollectionNameRe := regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") - - if strings.HasPrefix(collectionName, ".") { - return backends.NewError(backends.ErrorCodeCollectionStartsWithDot, nil) - } - - if !validateCollectionNameRe.MatchString(collectionName) || - !utf8.ValidString(collectionName) { - return backends.NewError(backends.ErrorCodeCollectionNameIsInvalid, nil) - } - - return db.CreateCollection(ctx, &backends.CreateCollectionParams{ - Name: collectionName, - }) -} From 53518ae60cb414a3317e859207beda46738cbedb Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 28 Jun 2023 13:08:29 +0200 Subject: [PATCH 31/66] improve prefix check --- internal/backends/sqlite/metadata/registry.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 1b7b8d9cd2fc..d8b651588685 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -40,7 +40,7 @@ const ( // Reserved prefix for SQLite table name. // https://www.sqlite.org/lang_createtable.html - reservedTablePrefix = "sqlite" + reservedTablePrefix = "sqlite_" // SQLite table name where FerretDB metadata is stored. metadataTableName = reservedPrefix + "collections" @@ -78,12 +78,14 @@ func collectionToTable(collectionName string) (string, error) { collectionName = strings.ToLower(collectionName) - // SQLite table cannot start with _sqlite prefix - if strings.HasPrefix(collectionName, reservedTablePrefix) { - collectionName = "_" + collectionName + tableName := collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)) + + // SQLite table cannot start with sqlite_ prefix + if strings.HasPrefix(tableName, reservedTablePrefix) { + tableName = "_" + tableName } - return collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)), nil + return tableName, nil } // DatabaseList returns a sorted list of existing databases. From 3b95fe114374041236329852d7ec78e171741d7a Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 28 Jun 2023 13:26:05 +0200 Subject: [PATCH 32/66] wip --- integration/basic_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 7906d97f4dc0..36b6ac38e5fb 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -15,6 +15,7 @@ package integration import ( + "errors" "fmt" "math" "strings" @@ -25,7 +26,6 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "go.mongodb.org/mongo-driver/x/mongo/driver" "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" @@ -524,7 +524,7 @@ func TestDatabaseName(t *testing.T) { ctx, collection := setup.Setup(t) err := collection.Database().Client().Database("").CreateCollection(ctx, collection.Name()) - expectedErr := driver.InvalidOperationError(driver.InvalidOperationError{MissingField: "Database"}) + expectedErr := errors.New("database name cannot be empty") assert.Equal(t, expectedErr, err) }) From b3d66ab2bb2084d9da65691d71be73ba4d4a9097 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 28 Jun 2023 13:40:22 +0200 Subject: [PATCH 33/66] Remove test --- integration/basic_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 36b6ac38e5fb..e86a4cad95fc 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -15,7 +15,6 @@ package integration import ( - "errors" "fmt" "math" "strings" @@ -518,16 +517,6 @@ func TestDatabaseName(t *testing.T) { } }) - t.Run("Empty", func(t *testing.T) { - t.Parallel() - - ctx, collection := setup.Setup(t) - - err := collection.Database().Client().Database("").CreateCollection(ctx, collection.Name()) - expectedErr := errors.New("database name cannot be empty") - assert.Equal(t, expectedErr, err) - }) - t.Run("63ok", func(t *testing.T) { ctx, collection := setup.Setup(t) From 1c2f423659cf98c1177ce2cb78fb690df2e20a11 Mon Sep 17 00:00:00 2001 From: noisersup Date: Wed, 28 Jun 2023 17:44:37 +0200 Subject: [PATCH 34/66] apply review comments --- internal/backends/sqlite/metadata/registry.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index d8b651588685..1154bac84c65 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -71,8 +71,7 @@ func (r *Registry) Close() { } // collectionToTable converts FerretDB collection name to SQLite table name. -// It returns ErrInvalidCollectionName error if the name is incorrect for SQLite backend. -func collectionToTable(collectionName string) (string, error) { +func collectionToTable(collectionName string) string { hash32 := fnv.New32a() must.NotFail(hash32.Write([]byte(collectionName))) @@ -85,7 +84,7 @@ func collectionToTable(collectionName string) (string, error) { tableName = "_" + tableName } - return tableName, nil + return tableName } // DatabaseList returns a sorted list of existing databases. @@ -169,10 +168,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - tableName, err := collectionToTable(collectionName) - if err != nil { - return false, err - } + tableName := collectionToTable(collectionName) // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 From c5fc06dca8d55de0283dfcf85b0c89f7b2390cee Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 29 Jun 2023 10:37:24 +0400 Subject: [PATCH 35/66] Update issue comment --- integration/basic_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index d1b4bd10a355..f9aecb440db4 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -416,7 +416,7 @@ func TestCollectionName(t *testing.T) { t.Skip(tc.skip) } - // TODO https://github.com/FerretDB/FerretDB/issues/2923 + // TODO https://github.com/FerretDB/FerretDB/issues/2747 // t.Parallel() err := collection.Database().CreateCollection(ctx, tc.collection) From 50694dffcaa421c68edfe0e75c99de443d0228b5 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 29 Jun 2023 10:50:50 +0400 Subject: [PATCH 36/66] Simplify --- internal/backends/sqlite/metadata/registry.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 1154bac84c65..2cf7c6231aaf 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -35,15 +35,12 @@ import ( ) const ( - // Reserved prefix for database and collection names. - reservedPrefix = "_ferretdb_" - - // Reserved prefix for SQLite table name. - // https://www.sqlite.org/lang_createtable.html + // This prefix is reserved by SQLite for internal use, + // see https://www.sqlite.org/lang_createtable.html. reservedTablePrefix = "sqlite_" // SQLite table name where FerretDB metadata is stored. - metadataTableName = reservedPrefix + "collections" + metadataTableName = "_ferretdb_collections" ) // Registry provides access to SQLite databases and collections information. @@ -75,11 +72,8 @@ func collectionToTable(collectionName string) string { hash32 := fnv.New32a() must.NotFail(hash32.Write([]byte(collectionName))) - collectionName = strings.ToLower(collectionName) - - tableName := collectionName + "_" + hex.EncodeToString(hash32.Sum(nil)) + tableName := strings.ToLower(collectionName) + "_" + hex.EncodeToString(hash32.Sum(nil)) - // SQLite table cannot start with sqlite_ prefix if strings.HasPrefix(tableName, reservedTablePrefix) { tableName = "_" + tableName } @@ -212,7 +206,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN if !rows.Next() { return "", backends.NewError( backends.ErrorCodeCollectionDoesNotExist, - fmt.Errorf("Collection %q does not exist", collectionName), + fmt.Errorf("collection %q does not exist", collectionName), ) } From a8bb52467855299073ead1d079e9512c823bd0eb Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 29 Jun 2023 11:56:42 +0200 Subject: [PATCH 37/66] rename CollectionGet --- internal/backends/sqlite/collection.go | 6 +++--- internal/backends/sqlite/metadata/registry.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index cd71b5ecdfc3..a4a96a58174a 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -53,7 +53,7 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } @@ -78,7 +78,7 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) // TODO https://github.com/FerretDB/FerretDB/issues/2750 - tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } @@ -125,7 +125,7 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Errorf("no database %q", c.dbName) } - tableName, err := c.r.CollectionGet(ctx, c.dbName, c.name) + tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 2cf7c6231aaf..ddf628809dde 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -186,10 +186,10 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return true, nil } -// CollectionGet returns table name associated with provided collection. +// GetTableName returns table name associated with provided collection. // // If database does not exist, no error is returned. -func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionName string) (string, error) { +func (r *Registry) GetTableName(ctx context.Context, dbName string, collectionName string) (string, error) { db := r.p.GetExisting(ctx, dbName) if db == nil { return "", nil @@ -228,7 +228,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - tableName, err := r.CollectionGet(ctx, dbName, collectionName) + tableName, err := r.GetTableName(ctx, dbName, collectionName) if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { return false, nil } From c81bc840cb7c8e93e8bb163a9532453c8f2ed1f5 Mon Sep 17 00:00:00 2001 From: noisersup Date: Thu, 29 Jun 2023 13:03:23 +0200 Subject: [PATCH 38/66] remove comment --- internal/backends/sqlite/metadata/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index ddf628809dde..4e52fcc1c25b 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -155,7 +155,6 @@ func (r *Registry) CollectionList(ctx context.Context, dbName string) ([]string, // // Returned boolean value indicates whether the collection was created. // If collection already exists, (false, nil) is returned. -// If collection name doesn't pass backend level validation it returns ErrInvalidCollectionName. func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collectionName string) (bool, error) { db, err := r.DatabaseGetOrCreate(ctx, dbName) if err != nil { From cb80f54d861390dff4d1666808ff7e41363c17a6 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 13:31:11 +0200 Subject: [PATCH 39/66] wip --- internal/backends/sqlite/collection.go | 16 +++++++++-- internal/backends/sqlite/metadata/registry.go | 28 ++++++++----------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index a4a96a58174a..3d92cf33d426 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -53,11 +53,17 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) + tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } + if !exists { + return &backends.QueryResult{ + Iter: newQueryIterator(ctx, nil), + }, nil + } + query := fmt.Sprintf(`SELECT _ferretdb_sjson FROM %q`, tableName) rows, err := db.QueryContext(ctx, query) @@ -78,7 +84,7 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) // TODO https://github.com/FerretDB/FerretDB/issues/2750 - tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) + tableName, _, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } @@ -125,11 +131,15 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Errorf("no database %q", c.dbName) } - tableName, err := c.r.GetTableName(ctx, c.dbName, c.name) + tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } + if !exists { + return nil, backends.NewError(backends.ErrorCodeCollectionDoesNotExist, fmt.Errorf("%s.%s does not exist", c.dbName, c.name)) + } + query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) var res backends.UpdateResult diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 4e52fcc1c25b..47bfd6ab17bd 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -28,7 +28,6 @@ import ( "modernc.org/sqlite" sqlitelib "modernc.org/sqlite/lib" - "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata/pool" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -187,34 +186,31 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti // GetTableName returns table name associated with provided collection. // -// If database does not exist, no error is returned. -func (r *Registry) GetTableName(ctx context.Context, dbName string, collectionName string) (string, error) { +// If database or collection does not exist, no error and false value is returned. +func (r *Registry) GetTableName(ctx context.Context, dbName string, collectionName string) (string, bool, error) { db := r.p.GetExisting(ctx, dbName) if db == nil { - return "", nil + return "", false, nil } query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) rows, err := db.QueryContext(ctx, query, collectionName) if err != nil { - return "", lazyerrors.Error(err) + return "", false, lazyerrors.Error(err) } defer rows.Close() if !rows.Next() { - return "", backends.NewError( - backends.ErrorCodeCollectionDoesNotExist, - fmt.Errorf("collection %q does not exist", collectionName), - ) + return "", false, nil } var name string if err = rows.Scan(&name); err != nil { - return "", lazyerrors.Error(err) + return "", false, lazyerrors.Error(err) } - return name, nil + return name, true, nil } // CollectionDrop drops a collection in the database. @@ -227,15 +223,15 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - tableName, err := r.GetTableName(ctx, dbName, collectionName) - if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { - return false, nil - } - + tableName, exists, err := r.GetTableName(ctx, dbName, collectionName) if err != nil { return false, lazyerrors.Error(err) } + if !exists { + return false, nil + } + // TODO use transactions // https://github.com/FerretDB/FerretDB/issues/2747 From a3b96490b5ee7abde4b2bd0efa680bf904fcfcd3 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 13:35:10 +0200 Subject: [PATCH 40/66] wip --- internal/backends/collection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 957261b98c31..c57a08f0d2e3 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -115,7 +115,7 @@ type UpdateResult struct { // Update updates documents in collection. func (cc *collectionContract) Update(ctx context.Context, params *UpdateParams) (res *UpdateResult, err error) { defer observability.FuncCall(ctx)() - defer checkError(err) + defer checkError(err, ErrorCodeCollectionDoesNotExist) res, err = cc.c.Update(ctx, params) return From 7c075bc877fa146a0d2c5be0d53f7ab00da3c4f4 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 13:43:22 +0200 Subject: [PATCH 41/66] wip --- internal/backends/sqlite/metadata/registry.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 47bfd6ab17bd..bb2e38bbf445 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -195,18 +195,14 @@ func (r *Registry) GetTableName(ctx context.Context, dbName string, collectionNa query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) - rows, err := db.QueryContext(ctx, query, collectionName) - if err != nil { - return "", false, lazyerrors.Error(err) - } - defer rows.Close() - - if !rows.Next() { - return "", false, nil - } + row := db.QueryRowContext(ctx, query, collectionName) var name string - if err = rows.Scan(&name); err != nil { + if err := row.Scan(&name); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + return "", false, lazyerrors.Error(err) } From 43c36ae4b1dd2359728ad2e827d0b46b56609481 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 13:45:53 +0200 Subject: [PATCH 42/66] wip --- internal/handlers/sqlite/msg_create.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 11650638ed11..95ff27547267 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -96,16 +96,12 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, } if !validateCollectionNameRe.MatchString(collectionName) || - !utf8.ValidString(collectionName) { + !utf8.ValidString(collectionName) || + strings.HasPrefix(collectionName, reservedPrefix) { msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") } - if strings.HasPrefix(collectionName, reservedPrefix) { - msg := fmt.Sprintf("Invalid collection name (%q is a reserved prefix): '%s.%s'", reservedPrefix, dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - } - db := h.b.Database(dbName) defer db.Close() From 22ff58dbe2aaf2c5c88aa29db1713dd372b90730 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 13:47:25 +0200 Subject: [PATCH 43/66] lint --- internal/backends/sqlite/collection.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 3d92cf33d426..8abc7a0d18f1 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -137,7 +137,10 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) } if !exists { - return nil, backends.NewError(backends.ErrorCodeCollectionDoesNotExist, fmt.Errorf("%s.%s does not exist", c.dbName, c.name)) + return nil, backends.NewError( + backends.ErrorCodeCollectionDoesNotExist, + fmt.Errorf("%s.%s does not exist", c.dbName, c.name), + ) } query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) From 6780ab3471a06f715dcd82111d9114b8b13d45bc Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 17:38:43 +0200 Subject: [PATCH 44/66] remove unused errorcode --- internal/backends/collection.go | 2 +- internal/backends/sqlite/collection.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index c57a08f0d2e3..957261b98c31 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -115,7 +115,7 @@ type UpdateResult struct { // Update updates documents in collection. func (cc *collectionContract) Update(ctx context.Context, params *UpdateParams) (res *UpdateResult, err error) { defer observability.FuncCall(ctx)() - defer checkError(err, ErrorCodeCollectionDoesNotExist) + defer checkError(err) res, err = cc.c.Update(ctx, params) return diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 8abc7a0d18f1..167b3573486d 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -137,10 +137,7 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) } if !exists { - return nil, backends.NewError( - backends.ErrorCodeCollectionDoesNotExist, - fmt.Errorf("%s.%s does not exist", c.dbName, c.name), - ) + return nil, fmt.Errorf("%s.%s does not exist", c.dbName, c.name) } query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) From 50397c1391a159d51ffd7625bb50f501018d4754 Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 17:49:05 +0200 Subject: [PATCH 45/66] remove empty collection error from update --- internal/backends/sqlite/collection.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 167b3573486d..fa1da5dadb22 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -136,14 +136,14 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Error(err) } + var res backends.UpdateResult + if !exists { - return nil, fmt.Errorf("%s.%s does not exist", c.dbName, c.name) + return &res, nil } query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE json_extract(_ferretdb_sjson, '$._id') = ?`, tableName) - var res backends.UpdateResult - iter := params.Docs.Iterator() defer iter.Close() From 1b1a1f4b4c7909b467a7ebbcf375b314d3b0e31d Mon Sep 17 00:00:00 2001 From: noisersup Date: Fri, 30 Jun 2023 18:04:55 +0200 Subject: [PATCH 46/66] cleanup --- internal/backends/sqlite/collection.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index e8e082516291..c8f3cef1a902 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -199,7 +199,14 @@ func (c *collection) Delete(ctx context.Context, params *backends.DeleteParams) return &backends.DeleteResult{Deleted: 0}, nil } - tableName := c.r.CollectionToTable(c.name) + tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) + if err != nil { + return nil, lazyerrors.Error(err) + } + + if !exists { + return &backends.DeleteResult{Deleted: 0}, nil + } query := fmt.Sprintf(`DELETE FROM %q WHERE _ferretdb_sjson -> '$._id' = ?`, tableName) From 5e519bd54dc28828a35fdbaceeb2c29b5c157da2 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 08:30:45 +0400 Subject: [PATCH 47/66] Tweak comment --- internal/backends/error.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/backends/error.go b/internal/backends/error.go index 5b99db72ff42..1e4223a7c847 100644 --- a/internal/backends/error.go +++ b/internal/backends/error.go @@ -74,6 +74,8 @@ func (err *Error) Error() string { } // ErrorCodeIs returns true if err is *Error with one of the given error codes. +// +// At least one error code must be given. func ErrorCodeIs(err error, code ErrorCode, codes ...ErrorCode) bool { e, ok := err.(*Error) //nolint:errorlint // do not inspect error chain if !ok { From 75706fdf02b6817f0ee68c96a6342c4f4bef43eb Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 13:22:39 +0400 Subject: [PATCH 48/66] Update comments --- internal/backends/collection.go | 4 ++-- internal/backends/sqlite/metadata/registry.go | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index b7cff88535d4..c149c06c19a5 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -84,8 +84,8 @@ func (cc *collectionContract) Query(ctx context.Context, params *QueryParams) (* // InsertParams represents the parameters of Collection.Insert method. type InsertParams struct { - // TODO that should be types.DocumentIterator - // https://github.com/FerretDB/FerretDB/issues/2750 + // TODO https://github.com/FerretDB/FerretDB/issues/2750 + // that should be types.DocumentIterator Iter iterator.Interface[int, any] } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index de6b15449e9a..278de64b2b72 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -101,7 +101,8 @@ func (r *Registry) DatabaseGetOrCreate(ctx context.Context, dbName string) (*sql return db, nil } - // TODO create unique indexes for name and table_name https://github.com/FerretDB/FerretDB/issues/2747 + // create unique indexes for name and table_name + // TODO https://github.com/FerretDB/FerretDB/issues/2747 _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %q (name, table_name, settings TEXT)", metadataTableName)) if err != nil { return nil, lazyerrors.Error(err) @@ -162,9 +163,8 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti tableName := collectionToTable(collectionName) - // TODO use transactions - // https://github.com/FerretDB/FerretDB/issues/2747 - + // use transactions + // TODO https://github.com/FerretDB/FerretDB/issues/2747 query := fmt.Sprintf("CREATE TABLE %q (_ferretdb_sjson TEXT)", tableName) if _, err = db.ExecContext(ctx, query); err != nil { var e *sqlite.Error @@ -228,9 +228,8 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - // TODO use transactions - // https://github.com/FerretDB/FerretDB/issues/2747 - + // use transactions + // TODO https://github.com/FerretDB/FerretDB/issues/2747 query := fmt.Sprintf("DELETE FROM %q WHERE name = ?", metadataTableName) if _, err := db.ExecContext(ctx, query, collectionName); err != nil { return false, lazyerrors.Error(err) From e4e77bf74737d14b2b02ce804c6a6a28e7a53d63 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 15:31:42 +0400 Subject: [PATCH 49/66] Add linter configuration --- .golangci.yml | 7 +++++++ integration/delete_test.go | 17 ++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c538a90ec506..33c23cb7feeb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,6 +51,13 @@ linters-settings: - "!**/cmd/envtool/*.go" deny: - pkg: github.com/FerretDB/FerretDB/internal/handlers/pg/pgdb + commonerrors: + files: + - $all + - "!**/internal/clientconn/*.go" + - "!**/internal/handlers/**/*.go" + deny: + - pkg: github.com/FerretDB/FerretDB/internal/handlers/commonerrors exhaustive: default-signifies-exhaustive: false gci: diff --git a/integration/delete_test.go b/integration/delete_test.go index 576acca5fe3e..6192ff4e32f0 100644 --- a/integration/delete_test.go +++ b/integration/delete_test.go @@ -24,7 +24,6 @@ import ( "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" - "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" ) // TestDeleteSimple checks simple cases of doc deletion. @@ -69,16 +68,16 @@ func TestDelete(t *testing.T) { "QueryNotSet": { deletes: bson.A{bson.D{}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrMissingField), - Name: commonerrors.ErrMissingField.String(), + Code: 40414, + Name: "Location40414", Message: "BSON field 'delete.deletes.q' is missing but a required field", }, }, "NotSet": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrMissingField), - Name: commonerrors.ErrMissingField.String(), + Code: 40414, + Name: "Location40414", Message: "BSON field 'delete.deletes.limit' is missing but a required field", }, }, @@ -92,8 +91,8 @@ func TestDelete(t *testing.T) { "InvalidFloat": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}, {"limit", 42.13}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrFailedToParse), - Name: commonerrors.ErrFailedToParse.String(), + Code: 9, + Name: "FailedToParse", Message: "The limit field in delete objects must be 0 or 1. Got 42.13", }, altMessage: "The 'delete.deletes.limit' field must be 0 or 1. Got 42.13", @@ -101,8 +100,8 @@ func TestDelete(t *testing.T) { "InvalidInt": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}, {"limit", 100}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrFailedToParse), - Name: commonerrors.ErrFailedToParse.String(), + Code: 9, + Name: "FailedToParse", Message: "The limit field in delete objects must be 0 or 1. Got 100", }, altMessage: "The 'delete.deletes.limit' field must be 0 or 1. Got 100", From eaf50a95fccc4df43fb1efdaeb9a016c63b2d50e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 16:17:20 +0400 Subject: [PATCH 50/66] Improve tests --- internal/backends/collection.go | 8 +- internal/backends/database.go | 2 +- internal/backends/error.go | 4 +- internal/backends/sqlite/backend.go | 1 - .../backends/sqlite/metadata/pool/pool.go | 48 +---- .../sqlite/metadata/pool/pool_test.go | 127 -------------- internal/backends/sqlite/metadata/pool/uri.go | 70 ++++++++ .../backends/sqlite/metadata/pool/uri_test.go | 165 ++++++++++++++++++ 8 files changed, 246 insertions(+), 179 deletions(-) create mode 100644 internal/backends/sqlite/metadata/pool/uri.go create mode 100644 internal/backends/sqlite/metadata/pool/uri_test.go diff --git a/internal/backends/collection.go b/internal/backends/collection.go index c149c06c19a5..c28189434e2c 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -68,7 +68,7 @@ type QueryResult struct { // Query executes a query against the collection. // -// If the collection does not exist it returns empty iterator. +// If database or collection does not exist it returns empty iterator. // // The passed context should be used for canceling the initial query. // It also can be used to close the returned iterator and free underlying resources, @@ -108,6 +108,8 @@ func (cc *collectionContract) Insert(ctx context.Context, params *InsertParams) // UpdateParams represents the parameters of Collection.Update method. type UpdateParams struct { + // TODO https://github.com/FerretDB/FerretDB/issues/3079 + // that should be types.DocumentIterator Docs *types.Array } @@ -117,6 +119,8 @@ type UpdateResult struct { } // Update updates documents in collection. +// +// Database or collection may not exist; that's not an error. func (cc *collectionContract) Update(ctx context.Context, params *UpdateParams) (*UpdateResult, error) { defer observability.FuncCall(ctx)() @@ -138,7 +142,7 @@ type DeleteResult struct { // Delete deletes documents in collection. // -// If requested database or collection does not exist it returns 0 deleted documents. +// Database or collection may not exist; that's not an error. func (cc *collectionContract) Delete(ctx context.Context, params *DeleteParams) (*DeleteResult, error) { defer observability.FuncCall(ctx)() diff --git a/internal/backends/database.go b/internal/backends/database.go index 6e7a04e7776d..71717b8f63ce 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -93,7 +93,7 @@ type CollectionInfo struct { // ListCollections returns information about collections in the database. // -// Database doesn't have to exist; that's not an error. +// Database may not exist; that's not an error. func (dbc *databaseContract) ListCollections(ctx context.Context, params *ListCollectionsParams) (*ListCollectionsResult, error) { defer observability.FuncCall(ctx)() diff --git a/internal/backends/error.go b/internal/backends/error.go index 1e4223a7c847..c60bbedf85b8 100644 --- a/internal/backends/error.go +++ b/internal/backends/error.go @@ -40,8 +40,8 @@ const ( // Error represents a backend error returned by all Backend, Database and Collection methods. type Error struct { - // this internal error can't be accessed by the caller; - // it may be nil + // This internal error can't be accessed by the caller; it exists only for debugging. + // It may be nil. err error code ErrorCode diff --git a/internal/backends/sqlite/backend.go b/internal/backends/sqlite/backend.go index 0020a881240d..d5c2f242b06f 100644 --- a/internal/backends/sqlite/backend.go +++ b/internal/backends/sqlite/backend.go @@ -18,7 +18,6 @@ import ( "context" "go.uber.org/zap" - _ "modernc.org/sqlite" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/backends/sqlite/metadata" diff --git a/internal/backends/sqlite/metadata/pool/pool.go b/internal/backends/sqlite/metadata/pool/pool.go index 6cf584926009..5a29e697667b 100644 --- a/internal/backends/sqlite/metadata/pool/pool.go +++ b/internal/backends/sqlite/metadata/pool/pool.go @@ -52,9 +52,9 @@ type Pool struct { token *resource.Token } -// New creates a pool for SQLite databases in the given directory. +// New creates a pool for SQLite databases in the directory specified by SQLite URI. func New(u string, l *zap.Logger) (*Pool, error) { - uri, err := validateURI(u) + uri, err := parseURI(u) if err != nil { return nil, fmt.Errorf("failed to parse SQLite URI %q: %s", u, err) } @@ -91,50 +91,6 @@ func New(u string, l *zap.Logger) (*Pool, error) { return p, nil } -// validateURI checks given URI value and returns parsed URL. -// URI should contain 'file' scheme and point to an existing directory. -// Path should end with '/'. Authority should be empty or absent. -// -// Returned URL contains path in both Path and Opaque to make String() method work correctly. -func validateURI(u string) (*url.URL, error) { - uri, err := url.Parse(u) - if err != nil { - return nil, err - } - - if uri.Scheme != "file" { - return nil, fmt.Errorf(`expected "file:" schema, got %q`, uri.Scheme) - } - - if uri.User != nil { - return nil, fmt.Errorf(`expected empty user info, got %q`, uri.User) - } - - if uri.Host != "" { - return nil, fmt.Errorf(`expected empty host, got %q`, uri.Host) - } - - if uri.Path == "" && uri.Opaque != "" { - uri.Path = uri.Opaque - } - uri.Opaque = uri.Path - - if !strings.HasSuffix(uri.Path, "/") { - return nil, fmt.Errorf(`expected path ending with "/", got %q`, uri.Path) - } - - fi, err := os.Stat(uri.Path) - if err != nil { - return nil, fmt.Errorf(`%q should be an existing directory, got %s`, uri.Path, err) - } - - if !fi.IsDir() { - return nil, fmt.Errorf(`%q should be an existing directory`, uri.Path) - } - - return uri, nil -} - // databaseName returns database name for given database file path. func (p *Pool) databaseName(databaseFile string) string { return strings.TrimSuffix(filepath.Base(databaseFile), filenameExtension) diff --git a/internal/backends/sqlite/metadata/pool/pool_test.go b/internal/backends/sqlite/metadata/pool/pool_test.go index fe5c6b037a71..7bcd5c7619eb 100644 --- a/internal/backends/sqlite/metadata/pool/pool_test.go +++ b/internal/backends/sqlite/metadata/pool/pool_test.go @@ -15,11 +15,8 @@ package pool import ( - "net/url" - "os" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/FerretDB/FerretDB/internal/util/testutil" @@ -58,127 +55,3 @@ func TestCreateDrop(t *testing.T) { db = p.GetExisting(ctx, t.Name()) require.Nil(t, db) } - -func TestNewBackend(t *testing.T) { - t.Parallel() - - err := os.MkdirAll("tmp/dir", 0o777) - require.NoError(t, err) - - _, err = os.Create("tmp/file") - require.NoError(t, err) - - t.Cleanup(func() { - err := os.RemoveAll("tmp") - require.NoError(t, err) - }) - - testCases := map[string]struct { - value string - uri *url.URL - err string - }{ - "LocalDirectory": { - value: "file:./", - uri: &url.URL{ - Scheme: "file", - Opaque: "./", - Path: "./", - }, - }, - "LocalSubDirectory": { - value: "file:./tmp/", - uri: &url.URL{ - Scheme: "file", - Opaque: "./tmp/", - Path: "./tmp/", - }, - }, - "LocalSubSubDirectory": { - value: "file:./tmp/dir/", - uri: &url.URL{ - Scheme: "file", - Opaque: "./tmp/dir/", - Path: "./tmp/dir/", - }, - }, - "LocalDirectoryWithParameters": { - value: "file:./tmp/?mode=ro", - uri: &url.URL{ - Scheme: "file", - Opaque: "./tmp/", - Path: "./tmp/", - RawQuery: "mode=ro", - }, - }, - "AbsoluteDirectory": { - value: "file:/tmp/", - uri: &url.URL{ - Scheme: "file", - Opaque: "/tmp/", - Path: "/tmp/", - OmitHost: true, - }, - }, - "WithEmptyAuthority": { - value: "file:///tmp/", - uri: &url.URL{ - Scheme: "file", - Opaque: "/tmp/", - Path: "/tmp/", - }, - }, - "WithEmptyAuthorityAndQuery": { - value: "file:///tmp/?mode=ro", - uri: &url.URL{ - Scheme: "file", - Opaque: "/tmp/", - Path: "/tmp/", - RawQuery: "mode=ro", - }, - }, - "HostIsNotEmpty": { - value: "file://localhost/./tmp/?mode=ro", - err: `expected empty host, got "localhost"`, - }, - "UserIsNotEmpty": { - value: "file://user:pass@./tmp/?mode=ro", - err: `expected empty user info, got "user:pass"`, - }, - "NoDirectory": { - value: "file:./nodir/", - err: `"./nodir/" should be an existing directory, got stat ./nodir/: no such file or directory`, - }, - "PathIsNotEndsWithSlash": { - value: "file:./tmp/file", - err: `expected path ending with "/", got "./tmp/file"`, - }, - "FileInsteadOfDirectory": { - value: "file:./tmp/file/", - err: `"./tmp/file/" should be an existing directory, got stat ./tmp/file/: not a directory`, - }, - "MalformedURI": { - value: ":./tmp/", - err: `parse ":./tmp/": missing protocol scheme`, - }, - "NoScheme": { - value: "./tmp/", - err: `expected "file:" schema, got ""`, - }, - } - for name, tc := range testCases { - name, tc := name, tc - t.Run(name, func(t *testing.T) { - t.Parallel() - - u, err := validateURI(tc.value) - if tc.err != "" { - assert.EqualError(t, err, tc.err) - return - } - - require.NoError(t, err) - assert.Equal(t, u, tc.uri) - }) - } -} diff --git a/internal/backends/sqlite/metadata/pool/uri.go b/internal/backends/sqlite/metadata/pool/uri.go new file mode 100644 index 000000000000..1ea96dc7b907 --- /dev/null +++ b/internal/backends/sqlite/metadata/pool/uri.go @@ -0,0 +1,70 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pool + +import ( + "fmt" + "net/url" + "os" + "strings" +) + +// parseURI checks given SQLite URI and returns a parsed form. +// +// URI should contain 'file' scheme and point to an existing directory. +// Path should end with '/'. Authority should be empty or absent. +// +// Returned URL contains path in both Path and Opaque to make String() method work correctly. +// Callers should use Path. +func parseURI(u string) (*url.URL, error) { + uri, err := url.Parse(u) + if err != nil { + return nil, err + } + + if uri.Scheme != "file" { + return nil, fmt.Errorf(`expected "file:" schema, got %q`, uri.Scheme) + } + + if uri.User != nil { + return nil, fmt.Errorf(`expected empty user info, got %q`, uri.User) + } + + if uri.Host != "" { + return nil, fmt.Errorf(`expected empty host, got %q`, uri.Host) + } + + if uri.Path == "" { + uri.Path = uri.Opaque + } + uri.Opaque = uri.Path + uri.RawPath = "" + uri.OmitHost = true + + if !strings.HasSuffix(uri.Path, "/") { + return nil, fmt.Errorf(`expected path ending with "/", got %q`, uri.Path) + } + + fi, err := os.Stat(uri.Path) + if err != nil { + return nil, fmt.Errorf(`%q should be an existing directory, got %s`, uri.Path, err) + } + + if !fi.IsDir() { + return nil, fmt.Errorf(`%q should be an existing directory`, uri.Path) + } + + return uri, nil +} diff --git a/internal/backends/sqlite/metadata/pool/uri_test.go b/internal/backends/sqlite/metadata/pool/uri_test.go new file mode 100644 index 000000000000..60346beba352 --- /dev/null +++ b/internal/backends/sqlite/metadata/pool/uri_test.go @@ -0,0 +1,165 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pool + +import ( + "net/url" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseURI(t *testing.T) { + t.Parallel() + + // tests rely on the fact that both ./tmp and /tmp exist + + err := os.MkdirAll("tmp/dir", 0o777) + require.NoError(t, err) + + _, err = os.Create("tmp/file") + require.NoError(t, err) + + t.Cleanup(func() { + err := os.RemoveAll("tmp") + require.NoError(t, err) + }) + + testCases := map[string]struct { + in string + uri *url.URL + out string // defaults to in + err string + }{ + "LocalDirectory": { + in: "file:./", + uri: &url.URL{ + Scheme: "file", + Opaque: "./", + Path: "./", + OmitHost: true, + }, + }, + "LocalSubDirectory": { + in: "file:./tmp/", + uri: &url.URL{ + Scheme: "file", + Opaque: "./tmp/", + Path: "./tmp/", + OmitHost: true, + }, + }, + "LocalSubSubDirectory": { + in: "file:./tmp/dir/", + uri: &url.URL{ + Scheme: "file", + Opaque: "./tmp/dir/", + Path: "./tmp/dir/", + OmitHost: true, + }, + }, + "LocalDirectoryWithParameters": { + in: "file:./tmp/?mode=ro", + uri: &url.URL{ + Scheme: "file", + Opaque: "./tmp/", + Path: "./tmp/", + OmitHost: true, + RawQuery: "mode=ro", + }, + }, + "AbsoluteDirectory": { + in: "file:/tmp/", + uri: &url.URL{ + Scheme: "file", + Opaque: "/tmp/", + Path: "/tmp/", + OmitHost: true, + }, + }, + "WithEmptyAuthority": { + in: "file:///tmp/", + uri: &url.URL{ + Scheme: "file", + Opaque: "/tmp/", + Path: "/tmp/", + OmitHost: true, + }, + out: "file:/tmp/", + }, + "WithEmptyAuthorityAndQuery": { + in: "file:///tmp/?mode=ro", + uri: &url.URL{ + Scheme: "file", + Opaque: "/tmp/", + Path: "/tmp/", + OmitHost: true, + RawQuery: "mode=ro", + }, + out: "file:/tmp/?mode=ro", + }, + "HostIsNotEmpty": { + in: "file://localhost/./tmp/?mode=ro", + err: `expected empty host, got "localhost"`, + }, + "UserIsNotEmpty": { + in: "file://user:pass@./tmp/?mode=ro", + err: `expected empty user info, got "user:pass"`, + }, + "NoDirectory": { + in: "file:./nodir/", + err: `"./nodir/" should be an existing directory, got stat ./nodir/: no such file or directory`, + }, + "PathIsNotEndsWithSlash": { + in: "file:./tmp/file", + err: `expected path ending with "/", got "./tmp/file"`, + }, + "FileInsteadOfDirectory": { + in: "file:./tmp/file/", + err: `"./tmp/file/" should be an existing directory, got stat ./tmp/file/: not a directory`, + }, + "MalformedURI": { + in: ":./tmp/", + err: `parse ":./tmp/": missing protocol scheme`, + }, + "NoScheme": { + in: "./tmp/", + err: `expected "file:" schema, got ""`, + }, + } + for name, tc := range testCases { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + u, err := parseURI(tc.in) + if tc.err != "" { + assert.EqualError(t, err, tc.err) + return + } + + require.NoError(t, err) + assert.Equal(t, u, tc.uri) + + out := tc.out + if out == "" { + out = tc.in + } + assert.Equal(t, out, u.String()) + }) + } +} From d95cca42c3a9f99d6e2f6560a9d8d95aca781482 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 16:58:20 +0400 Subject: [PATCH 51/66] Tweaks --- internal/backends/sqlite/metadata/pool/db.go | 7 +++++- .../backends/sqlite/metadata/pool/pool.go | 12 +++++---- internal/backends/sqlite/metadata/registry.go | 25 +++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/backends/sqlite/metadata/pool/db.go b/internal/backends/sqlite/metadata/pool/db.go index 12cce2b97d8a..0199ff805fd9 100644 --- a/internal/backends/sqlite/metadata/pool/db.go +++ b/internal/backends/sqlite/metadata/pool/db.go @@ -29,7 +29,12 @@ type db struct { token *resource.Token } -// openDB opens existing database. +// openDB opens existing database or creates a new one. +// +// All valid FerretDB database names are valid SQLite database names / file names, +// so no validation is needed. +// One exception is very long full path names for the filesystem, +// but we don't check it. func openDB(uri string) (*db, error) { sqlDB, err := sql.Open("sqlite", uri) if err != nil { diff --git a/internal/backends/sqlite/metadata/pool/pool.go b/internal/backends/sqlite/metadata/pool/pool.go index 5a29e697667b..bb21326e170a 100644 --- a/internal/backends/sqlite/metadata/pool/pool.go +++ b/internal/backends/sqlite/metadata/pool/pool.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package pool provides access to SQLite database connections. +// Package pool provides access to SQLite databases and their connections. // // It should be used only by the metadata package. package pool @@ -39,7 +39,7 @@ import ( // filenameExtension represents SQLite database filename extension. const filenameExtension = ".sqlite" -// Pool provides access to SQLite database connections. +// Pool provides access to SQLite databases and their connections. // //nolint:vet // for readability type Pool struct { @@ -53,6 +53,8 @@ type Pool struct { } // New creates a pool for SQLite databases in the directory specified by SQLite URI. +// +// All databases are opened on creation. func New(u string, l *zap.Logger) (*Pool, error) { uri, err := parseURI(u) if err != nil { @@ -135,7 +137,7 @@ func (p *Pool) List(ctx context.Context) []string { return res } -// GetExisting returns an existing database connection by name, or nil. +// GetExisting returns an existing database by valid name, or nil. func (p *Pool) GetExisting(ctx context.Context, name string) *sql.DB { p.rw.RLock() defer p.rw.RUnlock() @@ -148,9 +150,9 @@ func (p *Pool) GetExisting(ctx context.Context, name string) *sql.DB { return db.sqlDB } -// GetOrCreate returns an existing database connection by name, or creates a new one. +// GetOrCreate returns an existing database by valid name, or creates a new one. // -// Returned boolean value indicates whether the connection was created. +// Returned boolean value indicates whether the database was created. func (p *Pool) GetOrCreate(ctx context.Context, name string) (*sql.DB, bool, error) { sqlDB := p.GetExisting(ctx, name) if sqlDB != nil { diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 278de64b2b72..6bcb511d2b49 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -48,7 +48,7 @@ type Registry struct { l *zap.Logger } -// NewRegistry creates a registry for the given directory. +// NewRegistry creates a registry for SQLite databases in the directory specified by SQLite URI. func NewRegistry(u string, l *zap.Logger) (*Registry, error) { p, err := pool.New(u, l.Named("pool")) if err != nil { @@ -66,20 +66,6 @@ func (r *Registry) Close() { r.p.Close() } -// collectionToTable converts FerretDB collection name to SQLite table name. -func collectionToTable(collectionName string) string { - hash32 := fnv.New32a() - must.NotFail(hash32.Write([]byte(collectionName))) - - tableName := strings.ToLower(collectionName) + "_" + hex.EncodeToString(hash32.Sum(nil)) - - if strings.HasPrefix(tableName, reservedTablePrefix) { - tableName = "_" + tableName - } - - return tableName -} - // DatabaseList returns a sorted list of existing databases. func (r *Registry) DatabaseList(ctx context.Context) []string { return r.p.List(ctx) @@ -161,7 +147,14 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - tableName := collectionToTable(collectionName) + hash32 := fnv.New32a() + must.NotFail(hash32.Write([]byte(collectionName))) + + tableName := strings.ToLower(collectionName) + "_" + hex.EncodeToString(hash32.Sum(nil)) + + if strings.HasPrefix(tableName, reservedTablePrefix) { + tableName = "_" + tableName + } // use transactions // TODO https://github.com/FerretDB/FerretDB/issues/2747 From 5fef6b79a3101a8f62068f1e5ba285003e771fbc Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 17:57:25 +0400 Subject: [PATCH 52/66] Do not import `commonerrors` in tests --- .golangci.yml | 7 +++++++ integration/.golangci-new.yml | 2 +- integration/.golangci.yml | 8 ++++++++ integration/delete_test.go | 17 ++++++++--------- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c538a90ec506..33c23cb7feeb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,6 +51,13 @@ linters-settings: - "!**/cmd/envtool/*.go" deny: - pkg: github.com/FerretDB/FerretDB/internal/handlers/pg/pgdb + commonerrors: + files: + - $all + - "!**/internal/clientconn/*.go" + - "!**/internal/handlers/**/*.go" + deny: + - pkg: github.com/FerretDB/FerretDB/internal/handlers/commonerrors exhaustive: default-signifies-exhaustive: false gci: diff --git a/integration/.golangci-new.yml b/integration/.golangci-new.yml index 828391ae822e..e4f219c22878 100644 --- a/integration/.golangci-new.yml +++ b/integration/.golangci-new.yml @@ -49,6 +49,7 @@ linters: disable: # checked by the other configuration - asciicheck + - depguard - exhaustive - gci - goconst @@ -74,7 +75,6 @@ linters: - cyclop - deadcode - decorder - - depguard - dogsled - dupl - durationcheck diff --git a/integration/.golangci.yml b/integration/.golangci.yml index bb4e8237f6ab..c5264a730759 100644 --- a/integration/.golangci.yml +++ b/integration/.golangci.yml @@ -6,6 +6,13 @@ run: linters-settings: # asciicheck + depguard: + rules: + commonerrors: + files: + - $all + deny: + - pkg: github.com/FerretDB/FerretDB/internal/handlers/commonerrors exhaustive: default-signifies-exhaustive: false gci: @@ -96,6 +103,7 @@ linters: disable-all: true enable: - asciicheck + - depguard - exhaustive - gci - goconst diff --git a/integration/delete_test.go b/integration/delete_test.go index 576acca5fe3e..6192ff4e32f0 100644 --- a/integration/delete_test.go +++ b/integration/delete_test.go @@ -24,7 +24,6 @@ import ( "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" - "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" ) // TestDeleteSimple checks simple cases of doc deletion. @@ -69,16 +68,16 @@ func TestDelete(t *testing.T) { "QueryNotSet": { deletes: bson.A{bson.D{}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrMissingField), - Name: commonerrors.ErrMissingField.String(), + Code: 40414, + Name: "Location40414", Message: "BSON field 'delete.deletes.q' is missing but a required field", }, }, "NotSet": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrMissingField), - Name: commonerrors.ErrMissingField.String(), + Code: 40414, + Name: "Location40414", Message: "BSON field 'delete.deletes.limit' is missing but a required field", }, }, @@ -92,8 +91,8 @@ func TestDelete(t *testing.T) { "InvalidFloat": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}, {"limit", 42.13}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrFailedToParse), - Name: commonerrors.ErrFailedToParse.String(), + Code: 9, + Name: "FailedToParse", Message: "The limit field in delete objects must be 0 or 1. Got 42.13", }, altMessage: "The 'delete.deletes.limit' field must be 0 or 1. Got 42.13", @@ -101,8 +100,8 @@ func TestDelete(t *testing.T) { "InvalidInt": { deletes: bson.A{bson.D{{"q", bson.D{{"v", "foo"}}}, {"limit", 100}}}, err: &mongo.CommandError{ - Code: int32(commonerrors.ErrFailedToParse), - Name: commonerrors.ErrFailedToParse.String(), + Code: 9, + Name: "FailedToParse", Message: "The limit field in delete objects must be 0 or 1. Got 100", }, altMessage: "The 'delete.deletes.limit' field must be 0 or 1. Got 100", From c875dd1f4d8c2d679e2a542d3bfec585c237ba1b Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 19:20:04 +0400 Subject: [PATCH 53/66] More work --- internal/backends/sqlite/collection.go | 23 ++++---- internal/backends/sqlite/metadata/metadata.go | 36 ++++++++++++ internal/backends/sqlite/metadata/pool/uri.go | 3 + internal/backends/sqlite/metadata/registry.go | 57 ++++++++++++------- 4 files changed, 88 insertions(+), 31 deletions(-) create mode 100644 internal/backends/sqlite/metadata/metadata.go diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index c8f3cef1a902..1eaaeb2339e9 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -56,22 +56,21 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (* }, nil } - tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) + meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } - if !exists { + if meta == nil { return &backends.QueryResult{ Iter: newQueryIterator(ctx, nil), }, nil } - query := fmt.Sprintf(`SELECT _ferretdb_sjson FROM %q`, tableName) + query := fmt.Sprintf(`SELECT %s FROM %q`, metadata.DefaultColumn, meta.TableName) rows, err := db.QueryContext(ctx, query) if err != nil { - // No such table, return empty result. var e *sqlite.Error if errors.As(err, &e) && e.Code() == sqlitelib.SQLITE_ERROR { return &backends.QueryResult{Iter: newQueryIterator(ctx, nil)}, nil @@ -93,13 +92,13 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) // TODO https://github.com/FerretDB/FerretDB/issues/2750 - tableName, _, err := c.r.GetTableName(ctx, c.dbName, c.name) + meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } db := c.r.DatabaseGetExisting(ctx, c.dbName) - query := fmt.Sprintf(`INSERT INTO %q (_ferretdb_sjson) VALUES (?)`, tableName) + query := fmt.Sprintf(`INSERT INTO %q (%s) VALUES (?)`, meta.TableName, metadata.DefaultColumn) var res backends.InsertResult @@ -140,18 +139,18 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return nil, lazyerrors.Errorf("no database %q", c.dbName) } - tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) + meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } var res backends.UpdateResult - if !exists { + if meta == nil { return &res, nil } - query := fmt.Sprintf(`UPDATE %q SET _ferretdb_sjson = ? WHERE _ferretdb_sjson -> '$._id' = ?`, tableName) + query := fmt.Sprintf(`UPDATE %[1]q SET %s = ? WHERE %s = ?`, meta.TableName, metadata.DefaultColumn, metadata.IDColumn) iter := params.Docs.Iterator() defer iter.Close() @@ -199,16 +198,16 @@ func (c *collection) Delete(ctx context.Context, params *backends.DeleteParams) return &backends.DeleteResult{Deleted: 0}, nil } - tableName, exists, err := c.r.GetTableName(ctx, c.dbName, c.name) + meta, err := c.r.CollectionGet(ctx, c.dbName, c.name) if err != nil { return nil, lazyerrors.Error(err) } - if !exists { + if meta == nil { return &backends.DeleteResult{Deleted: 0}, nil } - query := fmt.Sprintf(`DELETE FROM %q WHERE _ferretdb_sjson -> '$._id' = ?`, tableName) + query := fmt.Sprintf(`DELETE FROM %q WHERE %s = ?`, meta.TableName, metadata.IDColumn) var deleted int64 diff --git a/internal/backends/sqlite/metadata/metadata.go b/internal/backends/sqlite/metadata/metadata.go new file mode 100644 index 000000000000..569041de1f95 --- /dev/null +++ b/internal/backends/sqlite/metadata/metadata.go @@ -0,0 +1,36 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metadata provides access to SQLite databases and collections information. +package metadata + +// Collection will probably have a method for getting column name / SQLite path expression for the given document field +// once we implement field extraction. +// IDColumn probably should go away. +// TODO https://github.com/FerretDB/FerretDB/issues/226 + +const ( + // IDColumn is a SQLite path expression for _id field. + IDColumn = "_ferretdb_sjson->'$._id'" + + // DefaultColumn is a column name for all fields expect _id. + DefaultColumn = "_ferretdb_sjson" +) + +// Collection represents collection metadata. +type Collection struct { + Name string + TableName string + Settings []byte +} diff --git a/internal/backends/sqlite/metadata/pool/uri.go b/internal/backends/sqlite/metadata/pool/uri.go index 1ea96dc7b907..d0c83bc2ab66 100644 --- a/internal/backends/sqlite/metadata/pool/uri.go +++ b/internal/backends/sqlite/metadata/pool/uri.go @@ -34,6 +34,9 @@ func parseURI(u string) (*url.URL, error) { return nil, err } + // handle mode=memory query parameter + // TODO https://github.com/FerretDB/FerretDB/issues/3084 + if uri.Scheme != "file" { return nil, fmt.Errorf(`expected "file:" schema, got %q`, uri.Scheme) } diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 6bcb511d2b49..666869c5637e 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package metadata provides access to SQLite databases and collections information. package metadata import ( @@ -55,6 +54,9 @@ func NewRegistry(u string, l *zap.Logger) (*Registry, error) { return nil, err } + // prefill cache + // TODO https://github.com/FerretDB/FerretDB/issues/2747 + return &Registry{ p: p, l: l, @@ -88,6 +90,7 @@ func (r *Registry) DatabaseGetOrCreate(ctx context.Context, dbName string) (*sql } // create unique indexes for name and table_name + // handle case when database and metadata table already exist // TODO https://github.com/FerretDB/FerretDB/issues/2747 _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %q (name, table_name, settings TEXT)", metadataTableName)) if err != nil { @@ -113,6 +116,9 @@ func (r *Registry) CollectionList(ctx context.Context, dbName string) ([]string, return nil, nil } + // use cache instead + // TODO https://github.com/FerretDB/FerretDB/issues/2747 + rows, err := db.QueryContext(ctx, fmt.Sprintf("SELECT name FROM %q ORDER BY name", metadataTableName)) if err != nil { return nil, lazyerrors.Error(err) @@ -147,18 +153,20 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - hash32 := fnv.New32a() - must.NotFail(hash32.Write([]byte(collectionName))) + // check cache first + // TODO https://github.com/FerretDB/FerretDB/issues/2747 - tableName := strings.ToLower(collectionName) + "_" + hex.EncodeToString(hash32.Sum(nil)) + h := fnv.New32a() + must.NotFail(h.Write([]byte(collectionName))) + tableName := strings.ToLower(collectionName) + "_" + hex.EncodeToString(h.Sum(nil)) if strings.HasPrefix(tableName, reservedTablePrefix) { tableName = "_" + tableName } // use transactions // TODO https://github.com/FerretDB/FerretDB/issues/2747 - query := fmt.Sprintf("CREATE TABLE %q (_ferretdb_sjson TEXT)", tableName) + query := fmt.Sprintf("CREATE TABLE %q (%s TEXT)", tableName, DefaultColumn) if _, err = db.ExecContext(ctx, query); err != nil { var e *sqlite.Error if errors.As(err, &e) && e.Code() == sqlitelib.SQLITE_ERROR { @@ -168,7 +176,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - query = fmt.Sprintf("INSERT INTO %q (name, table_name) VALUES (?, ?)", metadataTableName) + query = fmt.Sprintf("INSERT INTO %q (name, table_name, settings) VALUES (?, ?, '{}')", metadataTableName) if _, err = db.ExecContext(ctx, query, collectionName, tableName); err != nil { _, _ = db.ExecContext(ctx, fmt.Sprintf("DROP TABLE %q", tableName)) return false, lazyerrors.Error(err) @@ -177,29 +185,37 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return true, nil } -// GetTableName returns table name associated with provided collection. +// CollectionGet returns collection metadata. // -// If database or collection does not exist, no error and false value is returned. -func (r *Registry) GetTableName(ctx context.Context, dbName string, collectionName string) (string, bool, error) { +// If database or collection does not exist, (nil, nil) is returned. +func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionName string) (*Collection, error) { db := r.p.GetExisting(ctx, dbName) if db == nil { - return "", false, nil + return nil, nil } - query := fmt.Sprintf("SELECT table_name FROM %q WHERE name = ?", metadataTableName) + // check cache first + // TODO https://github.com/FerretDB/FerretDB/issues/2747 + + query := fmt.Sprintf("SELECT table_name, settings FROM %q WHERE name = ?", metadataTableName) row := db.QueryRowContext(ctx, query, collectionName) - var name string - if err := row.Scan(&name); err != nil { + var tableName string + var settings []byte + if err := row.Scan(&tableName, &settings); err != nil { if errors.Is(err, sql.ErrNoRows) { - return "", false, nil + return nil, nil } - return "", false, lazyerrors.Error(err) + return nil, lazyerrors.Error(err) } - return name, true, nil + return &Collection{ + Name: collectionName, + TableName: tableName, + Settings: settings, + }, nil } // CollectionDrop drops a collection in the database. @@ -212,12 +228,15 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, nil } - tableName, exists, err := r.GetTableName(ctx, dbName, collectionName) + // check cache first + // TODO https://github.com/FerretDB/FerretDB/issues/2747 + + info, err := r.CollectionGet(ctx, dbName, collectionName) if err != nil { return false, lazyerrors.Error(err) } - if !exists { + if info == nil { return false, nil } @@ -228,7 +247,7 @@ func (r *Registry) CollectionDrop(ctx context.Context, dbName string, collection return false, lazyerrors.Error(err) } - query = fmt.Sprintf("DROP TABLE %q", tableName) + query = fmt.Sprintf("DROP TABLE %q", info.TableName) if _, err := db.ExecContext(ctx, query); err != nil { return false, lazyerrors.Error(err) } From 14e2bfffca1c03b33b76481c262d1248dc5bd616 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 19:31:21 +0400 Subject: [PATCH 54/66] Add comment --- internal/backends/collection.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index c28189434e2c..e212d9f9922f 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -132,6 +132,7 @@ func (cc *collectionContract) Update(ctx context.Context, params *UpdateParams) // DeleteParams represents the parameters of Collection.Delete method. type DeleteParams struct { + // TODO https://github.com/FerretDB/FerretDB/issues/3085 IDs []any } From 67f909b88dd115f8d6a757d3720e9ba95a24fd3c Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 19 Jul 2023 20:38:12 +0400 Subject: [PATCH 55/66] Create unique index --- internal/backends/sqlite/metadata/registry.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 666869c5637e..70f878205bbc 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -176,6 +176,13 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } + indexName := tableName + "_id" + query = fmt.Sprintf("CREATE UNIQUE INDEX %q ON %q (%s)", indexName, tableName, IDColumn) + if _, err = db.ExecContext(ctx, query); err != nil { + _, _ = db.ExecContext(ctx, fmt.Sprintf("DROP TABLE %q", tableName)) + return false, lazyerrors.Error(err) + } + query = fmt.Sprintf("INSERT INTO %q (name, table_name, settings) VALUES (?, ?, '{}')", metadataTableName) if _, err = db.ExecContext(ctx, query, collectionName, tableName); err != nil { _, _ = db.ExecContext(ctx, fmt.Sprintf("DROP TABLE %q", tableName)) From b6d2961828f7f8813bb0006e82454b4fcd96f0f1 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 20 Jul 2023 11:47:12 +0400 Subject: [PATCH 56/66] Make SQLite URL configurable --- Taskfile.yml | 1 + integration/Taskfile.yml | 3 +++ integration/setup/listener.go | 9 +++---- integration/setup/setup.go | 1 + integration/setup/startup.go | 24 ++++++++++++------- .../sqlite/metadata/pool/pool_test.go | 3 ++- .../backends/sqlite/metadata/pool/uri_test.go | 14 +++++------ internal/backends/sqlite/metadata/registry.go | 4 ++-- 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 801dc18ab323..64aa83eb3e2a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -210,6 +210,7 @@ tasks: go test -count=1 -run='{{or .TEST_RUN .SHARD_RUN}}' -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverpkg=../... -coverprofile=integration-sqlite.txt . -target-backend=ferretdb-sqlite + -sqlite-url=file:../tmp/sqlite-tests/ -target-tls -compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=../build/certs/client.pem&tlsCaFile=../build/certs/rootCA-cert.pem' -disable-filter-pushdown diff --git a/integration/Taskfile.yml b/integration/Taskfile.yml index cc08682b9f6d..8122402d902d 100644 --- a/integration/Taskfile.yml +++ b/integration/Taskfile.yml @@ -27,6 +27,7 @@ tasks: go test -count=1 {{.RACE_FLAG}} -run=TestEnvData -tags=ferretdb_testenvdata . -target-backend=ferretdb-sqlite + -sqlite-url=file:../tmp/sqlite-tests/ # no hana yet - > go test -count=1 {{.RACE_FLAG}} -run=TestEnvData @@ -86,6 +87,7 @@ tasks: -log-level=error -bench-docs={{.BENCH_DOCS}} -target-backend=ferretdb-sqlite + -sqlite-url=file:../tmp/sqlite-tests/ | tee new-sqlite.txt - ../bin/benchstat{{exeExt}} old-sqlite.txt new-sqlite.txt @@ -98,6 +100,7 @@ tasks: -log-level=error -bench-docs={{.BENCH_DOCS}} -target-backend=ferretdb-sqlite + -sqlite-url=file:../tmp/sqlite-tests/ -- -disable-filter-pushdown | tee new-sqlite.txt - ../bin/benchstat{{exeExt}} old-sqlite.txt new-sqlite.txt diff --git a/integration/setup/listener.go b/integration/setup/listener.go index e30a110a0713..f361848d53eb 100644 --- a/integration/setup/listener.go +++ b/integration/setup/listener.go @@ -107,16 +107,19 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string switch *targetBackendF { case "ferretdb-pg": require.NotEmpty(tb, *postgreSQLURLF, "-postgresql-url must be set for %q", *targetBackendF) + require.Empty(tb, *sqliteURLF, "-sqlite-url must be empty for %q", *targetBackendF) require.Empty(tb, *hanaURLF, "-hana-url must be empty for %q", *targetBackendF) handler = "pg" case "ferretdb-sqlite": require.Empty(tb, *postgreSQLURLF, "-postgresql-url must be empty for %q", *targetBackendF) + require.NotEmpty(tb, *sqliteURLF, "-sqlite-url must be set for %q", *targetBackendF) require.Empty(tb, *hanaURLF, "-hana-url must be empty for %q", *targetBackendF) handler = "sqlite" case "ferretdb-hana": require.Empty(tb, *postgreSQLURLF, "-postgresql-url must be empty for %q", *targetBackendF) + require.Empty(tb, *sqliteURLF, "-sqlite-url must be empty for %q", *targetBackendF) require.NotEmpty(tb, *hanaURLF, "-hana-url must be set for %q", *targetBackendF) handler = "hana" @@ -137,10 +140,8 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string StateProvider: p, PostgreSQLURL: *postgreSQLURLF, - - SQLiteURL: sqliteURL.String(), - - HANAURL: *hanaURLF, + SQLiteURL: *sqliteURLF, + HANAURL: *hanaURLF, TestOpts: registry.TestOpts{ DisableFilterPushdown: *disableFilterPushdownF, diff --git a/integration/setup/setup.go b/integration/setup/setup.go index 3cb8c1140c9a..d24e68370e5b 100644 --- a/integration/setup/setup.go +++ b/integration/setup/setup.go @@ -48,6 +48,7 @@ var ( targetUnixSocketF = flag.Bool("target-unix-socket", false, "in-process FerretDB: use Unix socket") postgreSQLURLF = flag.String("postgresql-url", "", "in-process FerretDB: PostgreSQL URL for 'pg' handler.") + sqliteURLF = flag.String("sqlite-url", "", "in-process FerretDB: SQLite URI for 'sqlite' handler.") hanaURLF = flag.String("hana-url", "", "in-process FerretDB: Hana URL for 'hana' handler.") compatURLF = flag.String("compat-url", "", "compat system's (MongoDB) URL for compatibility tests; if empty, they are skipped") diff --git a/integration/setup/startup.go b/integration/setup/startup.go index 21826d96de7d..8804b454f4da 100644 --- a/integration/setup/startup.go +++ b/integration/setup/startup.go @@ -19,6 +19,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "runtime" "time" @@ -44,11 +45,6 @@ var listenerMetrics = connmetrics.NewListenerMetrics() // jaegerExporter is a shared Jaeger exporter for tests. var jaegerExporter *jaeger.Exporter -// sqliteURL is a URI for SQLite backend tests. -// -// We don't use testing.T.TempDir() or something to make debugging of failed tests easier. -var sqliteURL = must.NotFail(url.Parse("file:../tmp/sqlite-tests/")) - // Startup initializes things that should be initialized only once. func Startup() { logging.Setup(zap.DebugLevel, "") @@ -83,10 +79,20 @@ func Startup() { zap.S().Fatalf("Unknown target backend %q.", *targetBackendF) } - must.BeTrue(sqliteURL.Path == "") - must.BeTrue(sqliteURL.Opaque != "") - _ = os.Remove(sqliteURL.Opaque) - must.NoError(os.MkdirAll(sqliteURL.Opaque, 0o777)) + if *sqliteURLF != "" { + u, err := url.Parse(*sqliteURLF) + if err != nil { + zap.S().Fatalf("Failed to parse -sqlite-url %s: %s", u, err) + } + + must.BeTrue(u.Path == "") + must.BeTrue(u.Opaque != "") + + dir := must.NotFail(filepath.Abs(u.Opaque)) + zap.S().Infof("Recreating %s", dir) + _ = os.Remove(dir) + must.NoError(os.MkdirAll(dir, 0o777)) + } if u := *targetURLF; u != "" { client, err := makeClient(ctx, u) diff --git a/internal/backends/sqlite/metadata/pool/pool_test.go b/internal/backends/sqlite/metadata/pool/pool_test.go index 7bcd5c7619eb..7feaaa355508 100644 --- a/internal/backends/sqlite/metadata/pool/pool_test.go +++ b/internal/backends/sqlite/metadata/pool/pool_test.go @@ -25,7 +25,8 @@ import ( func TestCreateDrop(t *testing.T) { ctx := testutil.Ctx(t) - p, err := New("file:"+t.TempDir()+"/", testutil.Logger(t)) + // that also tests that query parameters are preserved + p, err := New("file:/?mode=memory", testutil.Logger(t)) require.NoError(t, err) defer p.Close() diff --git a/internal/backends/sqlite/metadata/pool/uri_test.go b/internal/backends/sqlite/metadata/pool/uri_test.go index 60346beba352..0e9ba89ebbc6 100644 --- a/internal/backends/sqlite/metadata/pool/uri_test.go +++ b/internal/backends/sqlite/metadata/pool/uri_test.go @@ -73,13 +73,13 @@ func TestParseURI(t *testing.T) { }, }, "LocalDirectoryWithParameters": { - in: "file:./tmp/?mode=ro", + in: "file:./tmp/?mode=memory", uri: &url.URL{ Scheme: "file", Opaque: "./tmp/", Path: "./tmp/", OmitHost: true, - RawQuery: "mode=ro", + RawQuery: "mode=memory", }, }, "AbsoluteDirectory": { @@ -102,22 +102,22 @@ func TestParseURI(t *testing.T) { out: "file:/tmp/", }, "WithEmptyAuthorityAndQuery": { - in: "file:///tmp/?mode=ro", + in: "file:///tmp/?mode=memory", uri: &url.URL{ Scheme: "file", Opaque: "/tmp/", Path: "/tmp/", OmitHost: true, - RawQuery: "mode=ro", + RawQuery: "mode=memory", }, - out: "file:/tmp/?mode=ro", + out: "file:/tmp/?mode=memory", }, "HostIsNotEmpty": { - in: "file://localhost/./tmp/?mode=ro", + in: "file://localhost/./tmp/?mode=memory", err: `expected empty host, got "localhost"`, }, "UserIsNotEmpty": { - in: "file://user:pass@./tmp/?mode=ro", + in: "file://user:pass@./tmp/?mode=memory", err: `expected empty user info, got "user:pass"`, }, "NoDirectory": { diff --git a/internal/backends/sqlite/metadata/registry.go b/internal/backends/sqlite/metadata/registry.go index 70f878205bbc..afb9bff15733 100644 --- a/internal/backends/sqlite/metadata/registry.go +++ b/internal/backends/sqlite/metadata/registry.go @@ -176,8 +176,7 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName string, collecti return false, lazyerrors.Error(err) } - indexName := tableName + "_id" - query = fmt.Sprintf("CREATE UNIQUE INDEX %q ON %q (%s)", indexName, tableName, IDColumn) + query = fmt.Sprintf("CREATE UNIQUE INDEX %q ON %q (%s)", tableName+"_id", tableName, IDColumn) if _, err = db.ExecContext(ctx, query); err != nil { _, _ = db.ExecContext(ctx, fmt.Sprintf("DROP TABLE %q", tableName)) return false, lazyerrors.Error(err) @@ -210,6 +209,7 @@ func (r *Registry) CollectionGet(ctx context.Context, dbName string, collectionN var tableName string var settings []byte + if err := row.Scan(&tableName, &settings); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil From 6a3545376283314eebf112e8a05efb177d6f676c Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 20 Jul 2023 19:43:40 +0400 Subject: [PATCH 57/66] Add validation --- internal/backends/backend.go | 24 +++-- internal/backends/database.go | 32 +++++-- internal/backends/doc.go | 2 +- internal/backends/error.go | 2 + internal/backends/errorcode_string.go | 12 +-- internal/backends/sqlite/backend.go | 4 +- internal/backends/sqlite/database.go | 4 +- internal/backends/validation.go | 55 ++++++++++++ internal/handlers/pg/pgdb/databases.go | 3 - internal/handlers/sqlite/msg_count.go | 25 +++++- internal/handlers/sqlite/msg_create.go | 42 +++------ internal/handlers/sqlite/msg_delete.go | 23 ++++- internal/handlers/sqlite/msg_drop.go | 17 +++- internal/handlers/sqlite/msg_dropdatabase.go | 2 + internal/handlers/sqlite/msg_find.go | 24 ++++- internal/handlers/sqlite/msg_insert.go | 24 ++++- .../handlers/sqlite/msg_listcollections.go | 13 ++- internal/handlers/sqlite/msg_ping.go | 20 +++-- internal/handlers/sqlite/msg_update.go | 44 ++++++--- internal/util/contract/contract.go | 58 ------------ internal/util/contract/contract_test.go | 89 ------------------- 21 files changed, 285 insertions(+), 234 deletions(-) create mode 100644 internal/backends/validation.go delete mode 100644 internal/util/contract/contract.go delete mode 100644 internal/util/contract/contract_test.go diff --git a/internal/backends/backend.go b/internal/backends/backend.go index 1b4fcf1a715c..23689ead1322 100644 --- a/internal/backends/backend.go +++ b/internal/backends/backend.go @@ -32,7 +32,7 @@ import ( // See backendContract and its methods for additional details. type Backend interface { Close() - Database(string) Database + Database(string) (Database, error) ListDatabases(context.Context, *ListDatabasesParams) (*ListDatabasesResult, error) DropDatabase(context.Context, *DropDatabaseParams) error @@ -70,9 +70,17 @@ func (bc *backendContract) Close() { // Database returns a Database instance for the given name. // -// The database does not need to exist; even parameters like name could be invalid. -func (bc *backendContract) Database(name string) Database { - return bc.b.Database(name) +// The database does not need to exist; even parameters like name could be invalid FIXME. +func (bc *backendContract) Database(name string) (Database, error) { + var res Database + err := validDatabaseName(name) + if err == nil { + res, err = bc.b.Database(name) + } + + checkError(err, ErrorCodeDatabaseNameIsInvalid) + + return res, err } // ListDatabasesParams represents the parameters of Backend.ListDatabases method. @@ -108,8 +116,12 @@ type DropDatabaseParams struct { func (bc *backendContract) DropDatabase(ctx context.Context, params *DropDatabaseParams) error { defer observability.FuncCall(ctx)() - err := bc.b.DropDatabase(ctx, params) - checkError(err, ErrorCodeDatabaseDoesNotExist) + err := validDatabaseName(params.Name) + if err == nil { + err = bc.b.DropDatabase(ctx, params) + } + + checkError(err, ErrorCodeDatabaseNameIsInvalid, ErrorCodeDatabaseDoesNotExist) return err } diff --git a/internal/backends/database.go b/internal/backends/database.go index 71717b8f63ce..34aed442521a 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -35,7 +35,7 @@ type Database interface { // TODO remove? Close() - Collection(string) Collection + Collection(string) (Collection, error) ListCollections(context.Context, *ListCollectionsParams) (*ListCollectionsResult, error) CreateCollection(context.Context, *CreateCollectionParams) error DropCollection(context.Context, *DropCollectionParams) error @@ -73,9 +73,17 @@ func (dbc *databaseContract) Close() { // Collection returns a Collection instance for the given name. // -// The collection (or database) does not need to exist; even parameters like name could be invalid. -func (dbc *databaseContract) Collection(name string) Collection { - return dbc.db.Collection(name) +// The collection (or database) does not need to exist; even parameters like name could be invalid FIXME. +func (dbc *databaseContract) Collection(name string) (Collection, error) { + var res Collection + err := validDatabaseName(name) + if err == nil { + res, err = dbc.db.Collection(name) + } + + checkError(err, ErrorCodeCollectionNameIsInvalid) + + return res, err } // ListCollectionsParams represents the parameters of Database.ListCollections method. @@ -114,8 +122,12 @@ type CreateCollectionParams struct { func (dbc *databaseContract) CreateCollection(ctx context.Context, params *CreateCollectionParams) error { defer observability.FuncCall(ctx)() - err := dbc.db.CreateCollection(ctx, params) - checkError(err, ErrorCodeCollectionAlreadyExists) + err := validDatabaseName(params.Name) + if err == nil { + err = dbc.db.CreateCollection(ctx, params) + } + + checkError(err, ErrorCodeCollectionNameIsInvalid, ErrorCodeCollectionAlreadyExists) return err } @@ -131,8 +143,12 @@ type DropCollectionParams struct { func (dbc *databaseContract) DropCollection(ctx context.Context, params *DropCollectionParams) error { defer observability.FuncCall(ctx)() - err := dbc.db.DropCollection(ctx, params) - checkError(err, ErrorCodeCollectionDoesNotExist) // TODO: ErrorCodeDatabaseDoesNotExist ? + err := validDatabaseName(params.Name) + if err == nil { + err = dbc.db.DropCollection(ctx, params) + } + + checkError(err, ErrorCodeCollectionNameIsInvalid, ErrorCodeCollectionDoesNotExist) // TODO: ErrorCodeDatabaseDoesNotExist ? return err } diff --git a/internal/backends/doc.go b/internal/backends/doc.go index 179b17b6e423..425c5227b13a 100644 --- a/internal/backends/doc.go +++ b/internal/backends/doc.go @@ -19,7 +19,7 @@ // 1. Interfaces are relatively high-level and "fat". // We are generally doing one backend interface call per handler call. // For example, `insert` command handler calls only -// `db.Database("database").Collection("collection").Insert(ctx, params)` method that would +// `db.Database("database").Collection("collection").Insert(ctx, params)` FIXME method that would // create a database if needed, create a collection if needed, and insert all documents with correct parameters. // There is no method to insert one document into an existing collection. // That shifts some complexity from a single handler into multiple backend implementations; diff --git a/internal/backends/error.go b/internal/backends/error.go index c60bbedf85b8..727ffa99d963 100644 --- a/internal/backends/error.go +++ b/internal/backends/error.go @@ -32,8 +32,10 @@ type ErrorCode int const ( _ ErrorCode = iota + ErrorCodeDatabaseNameIsInvalid ErrorCodeDatabaseDoesNotExist + ErrorCodeCollectionNameIsInvalid ErrorCodeCollectionDoesNotExist ErrorCodeCollectionAlreadyExists ) diff --git a/internal/backends/errorcode_string.go b/internal/backends/errorcode_string.go index 2444c28eb6c3..6ca7d3b1f679 100644 --- a/internal/backends/errorcode_string.go +++ b/internal/backends/errorcode_string.go @@ -8,14 +8,16 @@ func _() { // An "invalid array index" compiler error signifies that the constant values have changed. // Re-run the stringer command to generate them again. var x [1]struct{} - _ = x[ErrorCodeDatabaseDoesNotExist-1] - _ = x[ErrorCodeCollectionDoesNotExist-2] - _ = x[ErrorCodeCollectionAlreadyExists-3] + _ = x[ErrorCodeDatabaseNameIsInvalid-1] + _ = x[ErrorCodeDatabaseDoesNotExist-2] + _ = x[ErrorCodeCollectionNameIsInvalid-3] + _ = x[ErrorCodeCollectionDoesNotExist-4] + _ = x[ErrorCodeCollectionAlreadyExists-5] } -const _ErrorCode_name = "ErrorCodeDatabaseDoesNotExistErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExists" +const _ErrorCode_name = "ErrorCodeDatabaseNameIsInvalidErrorCodeDatabaseDoesNotExistErrorCodeCollectionNameIsInvalidErrorCodeCollectionDoesNotExistErrorCodeCollectionAlreadyExists" -var _ErrorCode_index = [...]uint8{0, 29, 60, 92} +var _ErrorCode_index = [...]uint8{0, 30, 59, 91, 122, 154} func (i ErrorCode) String() string { i -= 1 diff --git a/internal/backends/sqlite/backend.go b/internal/backends/sqlite/backend.go index d5c2f242b06f..1ef49f88caca 100644 --- a/internal/backends/sqlite/backend.go +++ b/internal/backends/sqlite/backend.go @@ -52,8 +52,8 @@ func (b *backend) Close() { } // Database implements backends.Backend interface. -func (b *backend) Database(name string) backends.Database { - return newDatabase(b.r, name) +func (b *backend) Database(name string) (backends.Database, error) { + return newDatabase(b.r, name), nil } // ListDatabases implements backends.Backend interface. diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 1fc160762973..d7d406b429ae 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -42,8 +42,8 @@ func (db *database) Close() { } // Collection implements backends.Database interface. -func (db *database) Collection(name string) backends.Collection { - return newCollection(db.r, db.name, name) +func (db *database) Collection(name string) (backends.Collection, error) { + return newCollection(db.r, db.name, name), nil } // ListCollections implements backends.Database interface. diff --git a/internal/backends/validation.go b/internal/backends/validation.go new file mode 100644 index 000000000000..cadbc35fbe46 --- /dev/null +++ b/internal/backends/validation.go @@ -0,0 +1,55 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backends + +import ( + "regexp" + "strings" + "unicode/utf8" +) + +// Reserved prefix for database and collection names. +const reservedPrefix = "_ferretdb_" + +// databaseNameRe validates FerretDB database name. +var databaseNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]{1,63}$") + +// collectionNameRe validates collection names. +// Empty collection name, names with `$` and `\x00`, +// or exceeding the 255 bytes limit are not allowed. +// Collection names that start with `.` are also not allowed. +var collectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") + +func validDatabaseName(name string) error { + if databaseNameRe.MatchString(name) { + return nil + } + + return NewError(ErrorCodeDatabaseNameIsInvalid, nil) +} + +func validCollectionName(name string) error { + if !collectionNameRe.MatchString(name) { + return NewError(ErrorCodeCollectionNameIsInvalid, nil) + } + if !utf8.ValidString(name) { + return NewError(ErrorCodeCollectionNameIsInvalid, nil) + } + if strings.HasPrefix(name, reservedPrefix) { + return NewError(ErrorCodeCollectionNameIsInvalid, nil) + } + + return nil +} diff --git a/internal/handlers/pg/pgdb/databases.go b/internal/handlers/pg/pgdb/databases.go index ba30acf5e516..5f31959d6505 100644 --- a/internal/handlers/pg/pgdb/databases.go +++ b/internal/handlers/pg/pgdb/databases.go @@ -24,7 +24,6 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" - "github.com/FerretDB/FerretDB/internal/util/contract" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" ) @@ -91,8 +90,6 @@ func CreateDatabaseIfNotExists(ctx context.Context, tx pgx.Tx, db string) error // DropDatabase drops FerretDB database. func DropDatabase(ctx context.Context, tx pgx.Tx, db string) (err error) { - defer contract.EnsureError(err, ErrSchemaNotExist) // if schema does not exist - if _, err = tx.Exec(ctx, `DROP SCHEMA `+pgx.Identifier{db}.Sanitize()+` CASCADE`); err == nil { return } diff --git a/internal/handlers/sqlite/msg_count.go b/internal/handlers/sqlite/msg_count.go index 891d5b77de9f..fffd1f991a95 100644 --- a/internal/handlers/sqlite/msg_count.go +++ b/internal/handlers/sqlite/msg_count.go @@ -17,8 +17,11 @@ package sqlite import ( "context" "errors" + "fmt" + "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" @@ -38,10 +41,28 @@ func (h *Handler) MsgCount(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, e return nil, err } - db := h.b.Database(params.DB) + db, err := h.b.Database(params.DB) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", params.DB) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "count") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() - queryRes, err := db.Collection(params.Collection).Query(ctx, nil) + c, err := db.Collection(params.Collection) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "count") + } + + return nil, lazyerrors.Error(err) + } + + queryRes, err := c.Query(ctx, nil) if err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index d619151f60b1..4b411cf3320d 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -17,9 +17,6 @@ package sqlite import ( "context" "fmt" - "regexp" - "strings" - "unicode/utf8" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" @@ -30,18 +27,6 @@ import ( "github.com/FerretDB/FerretDB/internal/wire" ) -// Reserved prefix for database and collection names. -const reservedPrefix = "_ferretdb_" - -// validateCollectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00`, -// or exceeding the 255 bytes limit are not allowed. -// Collection names that start with `.` are also not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") - -// validateDatabaseNameRe validates FerretDB database name. -var validateDatabaseNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]{1,63}$") - // MsgCreate implements HandlerInterface. func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, error) { document, err := msg.Document() @@ -93,24 +78,15 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, err } - if !validateDatabaseNameRe.MatchString(dbName) { - msg := fmt.Sprintf("Invalid namespace: %s.%s", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - } - - if strings.HasPrefix(collectionName, ".") { - msg := fmt.Sprintf("Collection names cannot start with '.': %s", collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") - } + db, err := h.b.Database(dbName) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", dbName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + } - if !validateCollectionNameRe.MatchString(collectionName) || - !utf8.ValidString(collectionName) || - strings.HasPrefix(collectionName, reservedPrefix) { - msg := fmt.Sprintf("Invalid collection name: '%s.%s'", dbName, collectionName) - return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + return nil, lazyerrors.Error(err) } - - db := h.b.Database(dbName) defer db.Close() err = db.CreateCollection(ctx, &backends.CreateCollectionParams{ @@ -128,6 +104,10 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return &reply, nil + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): + msg := fmt.Sprintf("Invalid collection name: %s.%s", dbName, collectionName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): msg := fmt.Sprintf("Collection %s.%s already exists.", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrNamespaceExists, msg, "create") diff --git a/internal/handlers/sqlite/msg_delete.go b/internal/handlers/sqlite/msg_delete.go index ad4cf4df2707..4b89aef4c6b7 100644 --- a/internal/handlers/sqlite/msg_delete.go +++ b/internal/handlers/sqlite/msg_delete.go @@ -17,6 +17,7 @@ package sqlite import ( "context" "errors" + "fmt" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" @@ -43,14 +44,30 @@ func (h *Handler) MsgDelete(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, var deleted int32 var delErrors commonerrors.WriteErrors - db := h.b.Database(params.DB) + db, err := h.b.Database(params.DB) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", params.DB) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "delete") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() - coll := db.Collection(params.Collection) + c, err := db.Collection(params.Collection) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "delete") + } + + return nil, lazyerrors.Error(err) + } // process every delete filter for i, deleteParams := range params.Deletes { - del, err := execDelete(ctx, coll, deleteParams.Filter, deleteParams.Limited) + del, err := execDelete(ctx, c, deleteParams.Filter, deleteParams.Limited) if err == nil { deleted += del continue diff --git a/internal/handlers/sqlite/msg_drop.go b/internal/handlers/sqlite/msg_drop.go index 88f6a88ae636..7aacf735ef20 100644 --- a/internal/handlers/sqlite/msg_drop.go +++ b/internal/handlers/sqlite/msg_drop.go @@ -16,6 +16,7 @@ package sqlite import ( "context" + "fmt" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" @@ -58,7 +59,15 @@ func (h *Handler) MsgDrop(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er } } - db := h.b.Database(dbName) + db, err := h.b.Database(dbName) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", dbName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "drop") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() err = db.DropCollection(ctx, &backends.DropCollectionParams{ @@ -78,8 +87,12 @@ func (h *Handler) MsgDrop(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er return &reply, nil + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): + msg := fmt.Sprintf("Invalid collection name: %s.%s", dbName, collectionName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "drop") + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist): - return nil, commonerrors.NewCommandErrorMsg(commonerrors.ErrNamespaceNotFound, "ns not found") + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrNamespaceNotFound, "ns not found", "drop") default: return nil, lazyerrors.Error(err) diff --git a/internal/handlers/sqlite/msg_dropdatabase.go b/internal/handlers/sqlite/msg_dropdatabase.go index 68b196a77275..82867071b030 100644 --- a/internal/handlers/sqlite/msg_dropdatabase.go +++ b/internal/handlers/sqlite/msg_dropdatabase.go @@ -59,6 +59,8 @@ func (h *Handler) MsgDropDatabase(ctx context.Context, msg *wire.OpMsg) (*wire.O switch { case err == nil: res.Set("dropped", dbName) + case backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid): + // nothing? case backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseDoesNotExist): // nothing default: diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index 78f8d565d2f6..a2ebc88913f9 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -17,8 +17,10 @@ package sqlite import ( "context" "errors" + "fmt" "time" + "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/clientconn/conninfo" "github.com/FerretDB/FerretDB/internal/clientconn/cursor" "github.com/FerretDB/FerretDB/internal/handlers/common" @@ -44,9 +46,27 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er username, _ := conninfo.Get(ctx).Auth() - db := h.b.Database(params.DB) + db, err := h.b.Database(params.DB) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", params.DB) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "find") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() + c, err := db.Collection(params.Collection) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "find") + } + + return nil, lazyerrors.Error(err) + } + cancel := func() {} if params.MaxTimeMS != 0 { // It is not clear if maxTimeMS affects only find, or both find and getMore (as the current code does). @@ -57,7 +77,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er // closer accumulates all things that should be closed / canceled. closer := iterator.NewMultiCloser(iterator.CloserFunc(cancel)) - queryRes, err := db.Collection(params.Collection).Query(ctx, nil) + queryRes, err := c.Query(ctx, nil) if err != nil { closer.Close() return nil, lazyerrors.Error(err) diff --git a/internal/handlers/sqlite/msg_insert.go b/internal/handlers/sqlite/msg_insert.go index e35047c9e412..adfc59d735cc 100644 --- a/internal/handlers/sqlite/msg_insert.go +++ b/internal/handlers/sqlite/msg_insert.go @@ -16,9 +16,11 @@ package sqlite import ( "context" + "fmt" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -37,13 +39,31 @@ func (h *Handler) MsgInsert(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, err } - db := h.b.Database(params.DB) + db, err := h.b.Database(params.DB) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", params.DB) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() + c, err := db.Collection(params.Collection) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") + } + + return nil, lazyerrors.Error(err) + } + iter := params.Docs.Iterator() defer iter.Close() - res, err := db.Collection(params.Collection).Insert(ctx, &backends.InsertParams{ + res, err := c.Insert(ctx, &backends.InsertParams{ Iter: iter, }) if err != nil { diff --git a/internal/handlers/sqlite/msg_listcollections.go b/internal/handlers/sqlite/msg_listcollections.go index f82f9e9dc8a9..9ef36e97fce0 100644 --- a/internal/handlers/sqlite/msg_listcollections.go +++ b/internal/handlers/sqlite/msg_listcollections.go @@ -16,8 +16,11 @@ package sqlite import ( "context" + "fmt" + "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/handlers/commonparams" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" @@ -52,7 +55,15 @@ func (h *Handler) MsgListCollections(ctx context.Context, msg *wire.OpMsg) (*wir } } - db := h.b.Database(dbName) + db, err := h.b.Database(dbName) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", dbName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "listCollections") + } + + return nil, lazyerrors.Error(err) + } defer db.Close() res, err := db.ListCollections(ctx, nil) diff --git a/internal/handlers/sqlite/msg_ping.go b/internal/handlers/sqlite/msg_ping.go index 2eabec38f01e..80ed78dd4cd8 100644 --- a/internal/handlers/sqlite/msg_ping.go +++ b/internal/handlers/sqlite/msg_ping.go @@ -16,8 +16,11 @@ package sqlite import ( "context" + "fmt" + "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" @@ -38,15 +41,20 @@ func (h *Handler) MsgPing(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er return nil, err } - db := h.b.Database(dbName) - defer db.Close() - - res, err := db.Collection("").Query(ctx, nil) + db, err := h.b.Database(dbName) if err != nil { - return nil, err + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", dbName) + return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "ping") + } + + return nil, lazyerrors.Error(err) } + defer db.Close() - res.Iter.Close() + if _, err := db.ListCollections(ctx, nil); err != nil { + return nil, lazyerrors.Error(err) + } must.NoError(reply.SetSections(wire.OpMsgSection{ Documents: []*types.Document{must.NotFail(types.NewDocument( diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index 57021874592c..6cc81a8e202a 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -17,9 +17,11 @@ package sqlite import ( "context" "errors" + "fmt" "github.com/FerretDB/FerretDB/internal/backends" "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" "github.com/FerretDB/FerretDB/internal/types" "github.com/FerretDB/FerretDB/internal/util/iterator" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" @@ -68,20 +70,42 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara var matched, modified int32 var upserted types.Array - db := h.b.Database(params.DB) - defer db.Close() + db, err := h.b.Database(params.DB) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { + msg := fmt.Sprintf("Invalid database name: %s", params.DB) + return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "update") + } - err := db.CreateCollection(ctx, &backends.CreateCollectionParams{Name: params.Collection}) - if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists) { - err = nil + return 0, 0, nil, lazyerrors.Error(err) } + defer db.Close() - if err != nil { + err = db.CreateCollection(ctx, &backends.CreateCollectionParams{Name: params.Collection}) + switch { + case err == nil: + // nothing + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): + // nothing + case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") + default: return 0, 0, nil, lazyerrors.Error(err) } for _, u := range params.Updates { - res, err := db.Collection(params.Collection).Query(ctx, nil) + c, err := db.Collection(params.Collection) + if err != nil { + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { + msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") + } + + return 0, 0, nil, lazyerrors.Error(err) + } + + res, err := c.Query(ctx, nil) if err != nil { return 0, 0, nil, lazyerrors.Error(err) } @@ -164,7 +188,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara iter := must.NotFail(types.NewArray(doc)).Iterator() defer iter.Close() - _, err = db.Collection(params.Collection).Insert(ctx, &backends.InsertParams{ + _, err = c.Insert(ctx, &backends.InsertParams{ Iter: iter, }) if err != nil { @@ -192,9 +216,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara continue } - updateRes, err := db. - Collection(params.Collection). - Update(ctx, &backends.UpdateParams{Docs: must.NotFail(types.NewArray(doc))}) + updateRes, err := c.Update(ctx, &backends.UpdateParams{Docs: must.NotFail(types.NewArray(doc))}) if err != nil { return 0, 0, nil, lazyerrors.Error(err) } diff --git a/internal/util/contract/contract.go b/internal/util/contract/contract.go deleted file mode 100644 index 035fc58f0a8d..000000000000 --- a/internal/util/contract/contract.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package contract provides Design by Contract functionality. -package contract - -import ( - "errors" - "fmt" - "reflect" - - "github.com/FerretDB/FerretDB/internal/util/debugbuild" -) - -// EnsureError checks that error is either nil or one of expected errors. -// -// If that is not the case, EnsureError panics in debug builds. -// It does nothing in non-debug builds. -func EnsureError(err error, expected ...error) { - if !debugbuild.Enabled { - return - } - - if err == nil { - return - } - - if reflect.ValueOf(err).IsZero() { - panic(fmt.Sprintf("EnsureError: invalid actual value %#v", err)) - } - - for _, target := range expected { - if target == nil { - panic(fmt.Sprintf("EnsureError: invalid expected value %#v", target)) - } - - if reflect.ValueOf(target).IsZero() { - panic(fmt.Sprintf("EnsureError: invalid expected value %#v", target)) - } - - if errors.Is(err, target) { - return - } - } - - panic(fmt.Sprintf("EnsureError: %#v", err)) -} diff --git a/internal/util/contract/contract_test.go b/internal/util/contract/contract_test.go deleted file mode 100644 index 2ead79724023..000000000000 --- a/internal/util/contract/contract_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2021 FerretDB Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package contract - -import ( - "fmt" - "io/fs" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/FerretDB/FerretDB/internal/util/debugbuild" -) - -var testError = fmt.Errorf("expected error value") - -func TestEnsureError(t *testing.T) { - t.Parallel() - - require.True(t, debugbuild.Enabled) - - assert.NotPanics(t, func() { - EnsureError(nil) - }) - - t.Run("Value", func(t *testing.T) { - t.Parallel() - - t.Run("OK", func(t *testing.T) { - t.Parallel() - - assert.NotPanics(t, func() { - var err error - EnsureError(err, testError) - }) - - assert.NotPanics(t, func() { - err := testError - EnsureError(err, testError) - }) - - assert.NotPanics(t, func() { - err := fmt.Errorf("wrapped: %w", testError) - EnsureError(err, testError) - }) - }) - - t.Run("Fail", func(t *testing.T) { - t.Parallel() - - assert.PanicsWithValue(t, `EnsureError: &errors.errorString{s:"some other error"}`, func() { - err := fmt.Errorf("some other error") - EnsureError(err, testError) - }) - }) - - t.Run("Invalid", func(t *testing.T) { - t.Parallel() - - assert.PanicsWithValue(t, `EnsureError: invalid actual value (*fs.PathError)(nil)`, func() { - var err *fs.PathError - EnsureError(err, testError) - }) - - assert.PanicsWithValue(t, `EnsureError: invalid expected value `, func() { - err := testError - EnsureError(err, nil) - }) - - assert.PanicsWithValue(t, `EnsureError: invalid expected value (*fs.PathError)(nil)`, func() { - err := testError - EnsureError(err, (*fs.PathError)(nil)) - }) - }) - }) -} From c5cc1d5fb9a744aca3259311d4253abaab40c441 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 10:28:16 +0400 Subject: [PATCH 58/66] Update comments --- internal/backends/backend.go | 2 +- internal/backends/database.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backends/backend.go b/internal/backends/backend.go index 23689ead1322..8004f5826643 100644 --- a/internal/backends/backend.go +++ b/internal/backends/backend.go @@ -70,7 +70,7 @@ func (bc *backendContract) Close() { // Database returns a Database instance for the given name. // -// The database does not need to exist; even parameters like name could be invalid FIXME. +// The database does not need to exist. func (bc *backendContract) Database(name string) (Database, error) { var res Database err := validDatabaseName(name) diff --git a/internal/backends/database.go b/internal/backends/database.go index 34aed442521a..4ec674da4851 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -73,7 +73,7 @@ func (dbc *databaseContract) Close() { // Collection returns a Collection instance for the given name. // -// The collection (or database) does not need to exist; even parameters like name could be invalid FIXME. +// The collection (or database) does not need to exist. func (dbc *databaseContract) Collection(name string) (Collection, error) { var res Collection err := validDatabaseName(name) From be93b2dff2866d1ab50128b7079016bae072151a Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 11:22:04 +0400 Subject: [PATCH 59/66] Tweak validation --- internal/backends/backend.go | 8 ++--- internal/backends/database.go | 12 ++++---- internal/backends/validation.go | 52 +++++++++++++++++++++++---------- 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/internal/backends/backend.go b/internal/backends/backend.go index 8004f5826643..7e972575c809 100644 --- a/internal/backends/backend.go +++ b/internal/backends/backend.go @@ -68,12 +68,12 @@ func (bc *backendContract) Close() { resource.Untrack(bc, bc.token) } -// Database returns a Database instance for the given name. +// Database returns a Database instance for the given valid name. // // The database does not need to exist. func (bc *backendContract) Database(name string) (Database, error) { var res Database - err := validDatabaseName(name) + err := validateDatabaseName(name) if err == nil { res, err = bc.b.Database(name) } @@ -112,11 +112,11 @@ type DropDatabaseParams struct { Name string } -// DropDatabase drops existing database for given parameters. +// DropDatabase drops existing database for given parameters (including valid name). func (bc *backendContract) DropDatabase(ctx context.Context, params *DropDatabaseParams) error { defer observability.FuncCall(ctx)() - err := validDatabaseName(params.Name) + err := validateDatabaseName(params.Name) if err == nil { err = bc.b.DropDatabase(ctx, params) } diff --git a/internal/backends/database.go b/internal/backends/database.go index 4ec674da4851..bc8007c92e25 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -71,12 +71,12 @@ func (dbc *databaseContract) Close() { resource.Untrack(dbc, dbc.token) } -// Collection returns a Collection instance for the given name. +// Collection returns a Collection instance for the given valid name. // // The collection (or database) does not need to exist. func (dbc *databaseContract) Collection(name string) (Collection, error) { var res Collection - err := validDatabaseName(name) + err := validateCollectionName(name) if err == nil { res, err = dbc.db.Collection(name) } @@ -116,13 +116,13 @@ type CreateCollectionParams struct { Name string } -// CreateCollection creates a new collection in the database; it should not already exist. +// CreateCollection creates a new collection with valid name in the database; it should not already exist. // // Database may or may not exist; it should be created automatically if needed. func (dbc *databaseContract) CreateCollection(ctx context.Context, params *CreateCollectionParams) error { defer observability.FuncCall(ctx)() - err := validDatabaseName(params.Name) + err := validateCollectionName(params.Name) if err == nil { err = dbc.db.CreateCollection(ctx, params) } @@ -137,13 +137,13 @@ type DropCollectionParams struct { Name string } -// DropCollection drops existing collection in the database. +// DropCollection drops existing collection with valid name in the database. // // The errors for non-existing database and non-existing collection are the same (TODO?). func (dbc *databaseContract) DropCollection(ctx context.Context, params *DropCollectionParams) error { defer observability.FuncCall(ctx)() - err := validDatabaseName(params.Name) + err := validateCollectionName(params.Name) if err == nil { err = dbc.db.DropCollection(ctx, params) } diff --git a/internal/backends/validation.go b/internal/backends/validation.go index cadbc35fbe46..c2b8bc9ef38d 100644 --- a/internal/backends/validation.go +++ b/internal/backends/validation.go @@ -20,34 +20,54 @@ import ( "unicode/utf8" ) -// Reserved prefix for database and collection names. -const reservedPrefix = "_ferretdb_" - -// databaseNameRe validates FerretDB database name. +// databaseNameRe validates database name. var databaseNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]{1,63}$") // collectionNameRe validates collection names. -// Empty collection name, names with `$` and `\x00`, -// or exceeding the 255 bytes limit are not allowed. -// Collection names that start with `.` are also not allowed. -var collectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") - -func validDatabaseName(name string) error { - if databaseNameRe.MatchString(name) { - return nil +var collectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") + +// validateDatabaseName checks that database name is valid for FerretDB. +// +// It follows MongoDB restrictions plus +// - allows only basic latin letters, digits, and basic punctuation; +// - disallows `_ferretdb_` prefix. +// +// That validation is quite restrictive because +// we expect it to be easy for users to change database names in their software/configuration if needed. +// +// Backends can do their own additional validation. +func validateDatabaseName(name string) error { + if !databaseNameRe.MatchString(name) { + return NewError(ErrorCodeDatabaseNameIsInvalid, nil) + } + + if strings.HasPrefix(name, "_ferretdb_") { + return NewError(ErrorCodeDatabaseNameIsInvalid, nil) } - return NewError(ErrorCodeDatabaseNameIsInvalid, nil) + return nil } -func validCollectionName(name string) error { +// validateCollectionName checks that collection name is valid for FerretDB. +// +// It follows MongoDB restrictions plus: +// - allows only UTF-8 characters; +// - disallows `_ferretdb_` prefix. +// +// That validation is quite lax because +// we expect it to be hard for users to change collection names in their software. +// +// Backends can do their own additional validation. +func validateCollectionName(name string) error { if !collectionNameRe.MatchString(name) { return NewError(ErrorCodeCollectionNameIsInvalid, nil) } - if !utf8.ValidString(name) { + + if strings.HasPrefix(name, "_ferretdb_") || strings.HasPrefix(name, "system.") { return NewError(ErrorCodeCollectionNameIsInvalid, nil) } - if strings.HasPrefix(name, reservedPrefix) { + + if !utf8.ValidString(name) { return NewError(ErrorCodeCollectionNameIsInvalid, nil) } From fe798661aac94115fbeaba99d4ff9a7a621c4f23 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 11:22:57 +0400 Subject: [PATCH 60/66] Fix number --- internal/handlers/pg/pgdb/collections.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/pg/pgdb/collections.go b/internal/handlers/pg/pgdb/collections.go index 6319c7483619..465747ed444d 100644 --- a/internal/handlers/pg/pgdb/collections.go +++ b/internal/handlers/pg/pgdb/collections.go @@ -35,7 +35,7 @@ import ( // validateCollectionNameRe validates collection names. // Empty collection name, names with `$` and `\x00`, -// or exceeding the 255 bytes limit are not allowed. +// or exceeding the 235 bytes limit are not allowed. // Collection names that start with `.` are also not allowed. var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") From 1c72efc362d215c0f23462dd07cffa8cb7d07594 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 15:50:09 +0400 Subject: [PATCH 61/66] Update comment --- internal/backends/doc.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/backends/doc.go b/internal/backends/doc.go index 425c5227b13a..4c0ee9a3025e 100644 --- a/internal/backends/doc.go +++ b/internal/backends/doc.go @@ -16,7 +16,7 @@ // // # Design principles. // -// 1. Interfaces are relatively high-level and "fat". +// 1. Interfaces are relatively high-level and "fat" (or not). // We are generally doing one backend interface call per handler call. // For example, `insert` command handler calls only // `db.Database("database").Collection("collection").Insert(ctx, params)` FIXME method that would @@ -33,4 +33,7 @@ // *Error values can't be wrapped or be present anywhere in the error chain. // Contracts enforce *Error codes; they are not documented in the code comments // but are visible in the contract's code (to avoid duplication). +// +// Figure it out, especially point number 1. Update, expand, etc. +// TODO https://github.com/FerretDB/FerretDB/issues/3069 package backends From cc083fa39390e27c11958bc91b7c3978d81af4c2 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 15:59:43 +0400 Subject: [PATCH 62/66] Update comments --- internal/backends/backend.go | 1 + internal/backends/collection.go | 3 ++- internal/backends/database.go | 1 + internal/backends/doc.go | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/backends/backend.go b/internal/backends/backend.go index 7e972575c809..1fd7650eb68b 100644 --- a/internal/backends/backend.go +++ b/internal/backends/backend.go @@ -37,6 +37,7 @@ type Backend interface { DropDatabase(context.Context, *DropDatabaseParams) error // There is no interface method to create a database; see package documentation. + // TODO https://github.com/FerretDB/FerretDB/issues/3069 } // backendContract implements Backend interface. diff --git a/internal/backends/collection.go b/internal/backends/collection.go index e212d9f9922f..ed8a5c5b5195 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -97,6 +97,7 @@ type InsertResult struct { // Insert inserts documents into the collection. // // Both database and collection may or may not exist; they should be created automatically if needed. +// TODO https://github.com/FerretDB/FerretDB/issues/3069 func (cc *collectionContract) Insert(ctx context.Context, params *InsertParams) (*InsertResult, error) { defer observability.FuncCall(ctx)() @@ -108,8 +109,8 @@ func (cc *collectionContract) Insert(ctx context.Context, params *InsertParams) // UpdateParams represents the parameters of Collection.Update method. type UpdateParams struct { - // TODO https://github.com/FerretDB/FerretDB/issues/3079 // that should be types.DocumentIterator + // TODO https://github.com/FerretDB/FerretDB/issues/3079 Docs *types.Array } diff --git a/internal/backends/database.go b/internal/backends/database.go index bc8007c92e25..0d054568944e 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -119,6 +119,7 @@ type CreateCollectionParams struct { // CreateCollection creates a new collection with valid name in the database; it should not already exist. // // Database may or may not exist; it should be created automatically if needed. +// TODO https://github.com/FerretDB/FerretDB/issues/3069 func (dbc *databaseContract) CreateCollection(ctx context.Context, params *CreateCollectionParams) error { defer observability.FuncCall(ctx)() diff --git a/internal/backends/doc.go b/internal/backends/doc.go index 4c0ee9a3025e..250138b1028a 100644 --- a/internal/backends/doc.go +++ b/internal/backends/doc.go @@ -19,7 +19,7 @@ // 1. Interfaces are relatively high-level and "fat" (or not). // We are generally doing one backend interface call per handler call. // For example, `insert` command handler calls only -// `db.Database("database").Collection("collection").Insert(ctx, params)` FIXME method that would +// `db.Database("database").Collection("collection").Insert(ctx, params)` method that would // create a database if needed, create a collection if needed, and insert all documents with correct parameters. // There is no method to insert one document into an existing collection. // That shifts some complexity from a single handler into multiple backend implementations; From 296973fcea3146c315002496a90034c3405c2a03 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 16:14:51 +0400 Subject: [PATCH 63/66] Tiny tweak --- internal/backends/sqlite/collection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 1eaaeb2339e9..bc9e2c3ab4cd 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -150,7 +150,7 @@ func (c *collection) Update(ctx context.Context, params *backends.UpdateParams) return &res, nil } - query := fmt.Sprintf(`UPDATE %[1]q SET %s = ? WHERE %s = ?`, meta.TableName, metadata.DefaultColumn, metadata.IDColumn) + query := fmt.Sprintf(`UPDATE %q SET %s = ? WHERE %s = ?`, meta.TableName, metadata.DefaultColumn, metadata.IDColumn) iter := params.Docs.Iterator() defer iter.Close() From 20f82004422474dd680705509f720f563eab1479 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 18:32:54 +0400 Subject: [PATCH 64/66] Fix tests --- integration/basic_test.go | 29 +++++++------------ integration/commands_administration_test.go | 1 - internal/backends/validation.go | 3 +- internal/handlers/pg/msg_create.go | 4 +-- internal/handlers/pg/msg_insert.go | 2 +- internal/handlers/pg/msg_update.go | 2 +- internal/handlers/pg/pgdb/collections.go | 2 +- internal/handlers/sqlite/msg_count.go | 4 +-- internal/handlers/sqlite/msg_create.go | 4 +-- internal/handlers/sqlite/msg_delete.go | 4 +-- internal/handlers/sqlite/msg_drop.go | 4 +-- internal/handlers/sqlite/msg_find.go | 4 +-- internal/handlers/sqlite/msg_insert.go | 4 +-- .../handlers/sqlite/msg_listcollections.go | 2 +- internal/handlers/sqlite/msg_ping.go | 2 +- internal/handlers/sqlite/msg_update.go | 6 ++-- 16 files changed, 34 insertions(+), 43 deletions(-) diff --git a/integration/basic_test.go b/integration/basic_test.go index 6dbdd2acb69f..3592decdf067 100644 --- a/integration/basic_test.go +++ b/integration/basic_test.go @@ -257,7 +257,7 @@ func TestCollectionName(t *testing.T) { collectionName300, ), }, - altMessage: fmt.Sprintf("Invalid collection name: 'TestCollectionName.%s'", collectionName300), + altMessage: fmt.Sprintf("Invalid collection name: %s", collectionName300), }, "LongEnough": { collection: collectionName235, @@ -272,7 +272,6 @@ func TestCollectionName(t *testing.T) { Code: 73, Message: `Invalid collection name: collection_name_with_a-$`, }, - altMessage: `Invalid collection name: 'TestCollectionName.collection_name_with_a-$'`, }, "WithADash": { collection: "collection_name_with_a-", @@ -287,7 +286,7 @@ func TestCollectionName(t *testing.T) { Code: 73, Message: "Invalid namespace specified 'TestCollectionName.'", }, - altMessage: "Invalid collection name: 'TestCollectionName.'", + altMessage: "Invalid collection name: ", }, "Null": { collection: "\x00", @@ -296,7 +295,7 @@ func TestCollectionName(t *testing.T) { Code: 73, Message: "namespaces cannot have embedded null characters", }, - altMessage: "Invalid collection name: 'TestCollectionName.\x00'", + altMessage: "Invalid collection name: \x00", }, "DotSurround": { collection: ".collection..", @@ -305,6 +304,7 @@ func TestCollectionName(t *testing.T) { Code: 73, Message: "Collection names cannot start with '.': .collection..", }, + altMessage: "Invalid collection name: .collection..", }, "Dot": { collection: "collection.name", @@ -347,14 +347,12 @@ func TestCollectionName(t *testing.T) { assert.NoError(t, err) - // check collection name is in the list. names, err := collection.Database().ListCollectionNames(ctx, bson.D{}) require.NoError(t, err) assert.Contains(t, names, tc.collection) newCollection := collection.Database().Collection(tc.collection) - // document can be inserted and found in the collection. doc := bson.D{{"_id", "item"}} _, err = newCollection.InsertOne(ctx, doc) require.NoError(t, err) @@ -423,15 +421,10 @@ func TestDatabaseName(t *testing.T) { "TooLongForBothDBs": { db: dbName64, err: &mongo.CommandError{ - Name: "InvalidNamespace", - Code: 73, - Message: fmt.Sprintf( - "Invalid namespace specified '%s.%s'", - dbName64, - "TestDatabaseName-Err", - ), + Name: "InvalidNamespace", + Code: 73, + Message: fmt.Sprintf("Invalid namespace specified '%s.TestDatabaseName-Err'", dbName64), }, - altMessage: fmt.Sprintf("Invalid namespace: %s.%s", dbName64, "TestDatabaseName-Err"), }, "WithASlash": { db: "/", @@ -440,17 +433,15 @@ func TestDatabaseName(t *testing.T) { Code: 73, Message: `Invalid namespace specified '/.TestDatabaseName-Err'`, }, - altMessage: `Invalid namespace: /.TestDatabaseName-Err`, }, "WithABackslash": { - db: "\\", + db: `\`, err: &mongo.CommandError{ Name: "InvalidNamespace", Code: 73, Message: `Invalid namespace specified '\.TestDatabaseName-Err'`, }, - altMessage: `Invalid namespace: \.TestDatabaseName-Err`, }, "WithADollarSign": { db: "name_with_a-$", @@ -459,6 +450,7 @@ func TestDatabaseName(t *testing.T) { Code: 73, Message: `Invalid namespace: name_with_a-$.TestDatabaseName-Err`, }, + altMessage: `Invalid namespace specified 'name_with_a-$.TestDatabaseName-Err'`, }, "WithSpace": { db: "data base", @@ -467,7 +459,6 @@ func TestDatabaseName(t *testing.T) { Code: 73, Message: `Invalid namespace specified 'data base.TestDatabaseName-Err'`, }, - altMessage: `Invalid namespace: data base.TestDatabaseName-Err`, }, "WithDot": { db: "database.test", @@ -476,7 +467,7 @@ func TestDatabaseName(t *testing.T) { Code: 73, Message: `'.' is an invalid character in the database name: database.test`, }, - altMessage: `Invalid namespace: database.test.TestDatabaseName-Err`, + altMessage: `Invalid namespace specified 'database.test.TestDatabaseName-Err'`, }, } diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 38d7f3e3b875..246457d04c96 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -803,7 +803,6 @@ func TestCommandsAdministrationDataSizeErrors(t *testing.T) { Name: "InvalidNamespace", Message: "Invalid namespace specified 'invalid'", }, - altMessage: "Invalid namespace specified 'invalid'", }, "InvalidNamespaceTypeDocument": { command: bson.D{{"dataSize", bson.D{}}}, diff --git a/internal/backends/validation.go b/internal/backends/validation.go index c2b8bc9ef38d..37e4391f5a35 100644 --- a/internal/backends/validation.go +++ b/internal/backends/validation.go @@ -24,7 +24,7 @@ import ( var databaseNameRe = regexp.MustCompile("^[a-zA-Z0-9_-]{1,63}$") // collectionNameRe validates collection names. -var collectionNameRe = regexp.MustCompile("^[^$\x00]{1,235}$") +var collectionNameRe = regexp.MustCompile("^[^\\.$\x00][^$\x00]{0,234}$") // validateDatabaseName checks that database name is valid for FerretDB. // @@ -52,6 +52,7 @@ func validateDatabaseName(name string) error { // // It follows MongoDB restrictions plus: // - allows only UTF-8 characters; +// - disallows '.' prefix (MongoDB fails to work with such collections correctly too); // - disallows `_ferretdb_` prefix. // // That validation is quite lax because diff --git a/internal/handlers/pg/msg_create.go b/internal/handlers/pg/msg_create.go index a0106fa529b3..d1bde7ed2cf9 100644 --- a/internal/handlers/pg/msg_create.go +++ b/internal/handlers/pg/msg_create.go @@ -98,11 +98,11 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return commonerrors.NewCommandErrorMsg(commonerrors.ErrNamespaceExists, msg) case errors.Is(err, pgdb.ErrInvalidDatabaseName): - msg := fmt.Sprintf("Invalid namespace: %s.%s", db, collection) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", db, collection) return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) case errors.Is(err, pgdb.ErrInvalidCollectionName): - msg := fmt.Sprintf("Invalid collection name: '%s.%s'", db, collection) + msg := fmt.Sprintf("Invalid collection name: %s", collection) return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) case errors.Is(err, pgdb.ErrCollectionStartsWithDot): diff --git a/internal/handlers/pg/msg_insert.go b/internal/handlers/pg/msg_insert.go index bbb4ca26c07b..00e0986a36cf 100644 --- a/internal/handlers/pg/msg_insert.go +++ b/internal/handlers/pg/msg_insert.go @@ -137,7 +137,7 @@ func insertDocument(ctx context.Context, tx pgx.Tx, qp *pgdb.QueryParams, doc *t return nil case errors.Is(err, pgdb.ErrInvalidCollectionName), errors.Is(err, pgdb.ErrInvalidDatabaseName): - msg := fmt.Sprintf("Invalid namespace: %s.%s", qp.DB, qp.Collection) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", qp.DB, qp.Collection) return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) case errors.Is(err, pgdb.ErrUniqueViolation): diff --git a/internal/handlers/pg/msg_update.go b/internal/handlers/pg/msg_update.go index 52d91f49ae0b..99a68321f8cb 100644 --- a/internal/handlers/pg/msg_update.go +++ b/internal/handlers/pg/msg_update.go @@ -56,7 +56,7 @@ func (h *Handler) MsgUpdate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, case err == nil: // do nothing case errors.Is(err, pgdb.ErrInvalidCollectionName), errors.Is(err, pgdb.ErrInvalidDatabaseName): - msg := fmt.Sprintf("Invalid namespace: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, document.Command()) default: return nil, lazyerrors.Error(err) diff --git a/internal/handlers/pg/pgdb/collections.go b/internal/handlers/pg/pgdb/collections.go index 465747ed444d..dd5035bc8c64 100644 --- a/internal/handlers/pg/pgdb/collections.go +++ b/internal/handlers/pg/pgdb/collections.go @@ -37,7 +37,7 @@ import ( // Empty collection name, names with `$` and `\x00`, // or exceeding the 235 bytes limit are not allowed. // Collection names that start with `.` are also not allowed. -var validateCollectionNameRe = regexp.MustCompile("^[^.$\x00][^$\x00]{0,234}$") +var validateCollectionNameRe = regexp.MustCompile("^[^\\.$\x00][^$\x00]{0,234}$") // Collections returns a sorted list of FerretDB collection names. // diff --git a/internal/handlers/sqlite/msg_count.go b/internal/handlers/sqlite/msg_count.go index fffd1f991a95..10c7056f98ab 100644 --- a/internal/handlers/sqlite/msg_count.go +++ b/internal/handlers/sqlite/msg_count.go @@ -44,7 +44,7 @@ func (h *Handler) MsgCount(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, e db, err := h.b.Database(params.DB) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", params.DB) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "count") } @@ -55,7 +55,7 @@ func (h *Handler) MsgCount(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, e c, err := db.Collection(params.Collection) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "count") } diff --git a/internal/handlers/sqlite/msg_create.go b/internal/handlers/sqlite/msg_create.go index 4b411cf3320d..c30285a29cea 100644 --- a/internal/handlers/sqlite/msg_create.go +++ b/internal/handlers/sqlite/msg_create.go @@ -81,7 +81,7 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, db, err := h.b.Database(dbName) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", dbName) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", dbName, collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") } @@ -105,7 +105,7 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return &reply, nil case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): - msg := fmt.Sprintf("Invalid collection name: %s.%s", dbName, collectionName) + msg := fmt.Sprintf("Invalid collection name: %s", collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "create") case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): diff --git a/internal/handlers/sqlite/msg_delete.go b/internal/handlers/sqlite/msg_delete.go index 4b89aef4c6b7..4c9faf4ffade 100644 --- a/internal/handlers/sqlite/msg_delete.go +++ b/internal/handlers/sqlite/msg_delete.go @@ -47,7 +47,7 @@ func (h *Handler) MsgDelete(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, db, err := h.b.Database(params.DB) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", params.DB) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "delete") } @@ -58,7 +58,7 @@ func (h *Handler) MsgDelete(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, c, err := db.Collection(params.Collection) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "delete") } diff --git a/internal/handlers/sqlite/msg_drop.go b/internal/handlers/sqlite/msg_drop.go index 7aacf735ef20..cbc2aa238eb6 100644 --- a/internal/handlers/sqlite/msg_drop.go +++ b/internal/handlers/sqlite/msg_drop.go @@ -62,7 +62,7 @@ func (h *Handler) MsgDrop(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er db, err := h.b.Database(dbName) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", dbName) + msg := fmt.Sprintf("Invalid namespace specified '%s'", dbName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "drop") } @@ -88,7 +88,7 @@ func (h *Handler) MsgDrop(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er return &reply, nil case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): - msg := fmt.Sprintf("Invalid collection name: %s.%s", dbName, collectionName) + msg := fmt.Sprintf("Invalid collection name: %s", collectionName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "drop") case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist): diff --git a/internal/handlers/sqlite/msg_find.go b/internal/handlers/sqlite/msg_find.go index a2ebc88913f9..8ab545fd6016 100644 --- a/internal/handlers/sqlite/msg_find.go +++ b/internal/handlers/sqlite/msg_find.go @@ -49,7 +49,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er db, err := h.b.Database(params.DB) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", params.DB) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "find") } @@ -60,7 +60,7 @@ func (h *Handler) MsgFind(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er c, err := db.Collection(params.Collection) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "find") } diff --git a/internal/handlers/sqlite/msg_insert.go b/internal/handlers/sqlite/msg_insert.go index adfc59d735cc..ef40a4cec1a8 100644 --- a/internal/handlers/sqlite/msg_insert.go +++ b/internal/handlers/sqlite/msg_insert.go @@ -42,7 +42,7 @@ func (h *Handler) MsgInsert(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, db, err := h.b.Database(params.DB) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", params.DB) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") } @@ -53,7 +53,7 @@ func (h *Handler) MsgInsert(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, c, err := db.Collection(params.Collection) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") } diff --git a/internal/handlers/sqlite/msg_listcollections.go b/internal/handlers/sqlite/msg_listcollections.go index 9ef36e97fce0..163dea5eb3d1 100644 --- a/internal/handlers/sqlite/msg_listcollections.go +++ b/internal/handlers/sqlite/msg_listcollections.go @@ -58,7 +58,7 @@ func (h *Handler) MsgListCollections(ctx context.Context, msg *wire.OpMsg) (*wir db, err := h.b.Database(dbName) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", dbName) + msg := fmt.Sprintf("Invalid namespace specified '%s'", dbName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "listCollections") } diff --git a/internal/handlers/sqlite/msg_ping.go b/internal/handlers/sqlite/msg_ping.go index 80ed78dd4cd8..2d69ba60eddc 100644 --- a/internal/handlers/sqlite/msg_ping.go +++ b/internal/handlers/sqlite/msg_ping.go @@ -44,7 +44,7 @@ func (h *Handler) MsgPing(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er db, err := h.b.Database(dbName) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", dbName) + msg := fmt.Sprintf("Invalid namespace specified '%s'", dbName) return nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "ping") } diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index 6cc81a8e202a..e17084cd851c 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -73,7 +73,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara db, err := h.b.Database(params.DB) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseNameIsInvalid) { - msg := fmt.Sprintf("Invalid database name: %s", params.DB) + msg := fmt.Sprintf("Invalid namespace specified '%s.%s'", params.DB, params.Collection) return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "update") } @@ -88,7 +88,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionAlreadyExists): // nothing case backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid): - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") default: return 0, 0, nil, lazyerrors.Error(err) @@ -98,7 +98,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara c, err := db.Collection(params.Collection) if err != nil { if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionNameIsInvalid) { - msg := fmt.Sprintf("Invalid collection name: %s.%s", params.DB, params.Collection) + msg := fmt.Sprintf("Invalid collection name: %s", params.Collection) return 0, 0, nil, commonerrors.NewCommandErrorMsgWithArgument(commonerrors.ErrInvalidNamespace, msg, "insert") } From c39f316b800a8547070cc06e652709e8e7c27930 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 18:37:33 +0400 Subject: [PATCH 65/66] Lint --- internal/backends/backend.go | 1 + internal/backends/database.go | 1 + internal/handlers/sqlite/msg_update.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/backends/backend.go b/internal/backends/backend.go index 1fd7650eb68b..766318a7355e 100644 --- a/internal/backends/backend.go +++ b/internal/backends/backend.go @@ -74,6 +74,7 @@ func (bc *backendContract) Close() { // The database does not need to exist. func (bc *backendContract) Database(name string) (Database, error) { var res Database + err := validateDatabaseName(name) if err == nil { res, err = bc.b.Database(name) diff --git a/internal/backends/database.go b/internal/backends/database.go index 0d054568944e..ebfe151c650d 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -76,6 +76,7 @@ func (dbc *databaseContract) Close() { // The collection (or database) does not need to exist. func (dbc *databaseContract) Collection(name string) (Collection, error) { var res Collection + err := validateCollectionName(name) if err == nil { res, err = dbc.db.Collection(name) diff --git a/internal/handlers/sqlite/msg_update.go b/internal/handlers/sqlite/msg_update.go index e17084cd851c..d42c3580776e 100644 --- a/internal/handlers/sqlite/msg_update.go +++ b/internal/handlers/sqlite/msg_update.go @@ -82,6 +82,7 @@ func (h *Handler) updateDocument(ctx context.Context, params *common.UpdatesPara defer db.Close() err = db.CreateCollection(ctx, &backends.CreateCollectionParams{Name: params.Collection}) + switch { case err == nil: // nothing From e266e35b1219af0498dfca730c30cfbbdd7f698d Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 26 Jul 2023 18:51:16 +0400 Subject: [PATCH 66/66] Add explicit panic --- internal/backends/sqlite/collection.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index bc9e2c3ab4cd..1cf76ee0c95d 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -97,6 +97,10 @@ func (c *collection) Insert(ctx context.Context, params *backends.InsertParams) return nil, lazyerrors.Error(err) } + if meta == nil { + panic(fmt.Sprintf("just created collection %q does not exist", c.name)) + } + db := c.r.DatabaseGetExisting(ctx, c.dbName) query := fmt.Sprintf(`INSERT INTO %q (%s) VALUES (?)`, meta.TableName, metadata.DefaultColumn)