From c0fdb8e69aea95a62fa1fd4eb4f92b01a135a361 Mon Sep 17 00:00:00 2001 From: Lorant Polya Date: Mon, 29 May 2023 15:54:54 -0700 Subject: [PATCH 1/6] Hana Insert Impl Basics --- internal/handlers/hana/hanadb/collections.go | 27 +++- internal/handlers/hana/hanadb/databases.go | 29 ++++- internal/handlers/hana/hanadb/insert.go | 57 +++++++++ internal/handlers/hana/hanadb/query.go | 21 ++++ internal/handlers/hana/msg_create.go | 9 +- internal/handlers/hana/msg_drop.go | 7 +- internal/handlers/hana/msg_dropdatabase.go | 6 +- internal/handlers/hana/msg_insert.go | 124 ++++++++++++++++++- 8 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 internal/handlers/hana/hanadb/insert.go create mode 100644 internal/handlers/hana/hanadb/query.go diff --git a/internal/handlers/hana/hanadb/collections.go b/internal/handlers/hana/hanadb/collections.go index c2ca1dc6ddb9..158b60d30521 100644 --- a/internal/handlers/hana/hanadb/collections.go +++ b/internal/handlers/hana/hanadb/collections.go @@ -16,25 +16,40 @@ package hanadb import ( "context" + "errors" "fmt" ) // CreateCollection creates a new SAP HANA JSON Document Store collection. // // It returns ErrAlreadyExist if collection already exist. -func (hanaPool *Pool) CreateCollection(ctx context.Context, db, collection string) error { - sql := fmt.Sprintf("CREATE COLLECTION %q.%q", db, collection) +func (hanaPool *Pool) CreateCollection(ctx context.Context, qp *QueryParams) error { + sql := fmt.Sprintf("CREATE COLLECTION %q.%q", qp.DB, qp.Collection) _, err := hanaPool.ExecContext(ctx, sql) return getHanaErrorIfExists(err) } -// DropCollection drops collection +// CreateCollectionIfNotExists creates a new SAP HANA JSON Document Store collection. // -// It returns ErrTableNotExist is collection does not exist. -func (hanaPool *Pool) DropCollection(ctx context.Context, db, collection string) error { - sql := fmt.Sprintf("DROP COLLECTION %q.%q", db, collection) +// Returns nil if collection already exist. +func (hanaPool *Pool) CreateCollectionIfNotExists(ctx context.Context, qp *QueryParams) error { + err := hanaPool.CreateCollection(ctx, qp) + + switch { + case errors.Is(err, ErrCollectionAlreadyExist): + return nil + default: + return err + } +} + +// DropCollection drops collection. +// +// Returns ErrTableNotExist is collection does not exist. +func (hanaPool *Pool) DropCollection(ctx context.Context, qp *QueryParams) error { + sql := fmt.Sprintf("DROP COLLECTION %q.%q", qp.DB, qp.Collection) _, err := hanaPool.ExecContext(ctx, sql) diff --git a/internal/handlers/hana/hanadb/databases.go b/internal/handlers/hana/hanadb/databases.go index 5ac55dd53e4e..614191a2be84 100644 --- a/internal/handlers/hana/hanadb/databases.go +++ b/internal/handlers/hana/hanadb/databases.go @@ -16,23 +16,40 @@ package hanadb import ( "context" + "errors" "fmt" ) // CreateSchema creates a schema in SAP HANA JSON Document Store. -func (hanaPool *Pool) CreateSchema(ctx context.Context, db string) error { - sqlStmt := fmt.Sprintf("CREATE SCHEMA %q", db) +// +// Returns ErrSchemaAlreadyExist if schema already exists. +func (hanaPool *Pool) CreateSchema(ctx context.Context, qp *QueryParams) error { + sqlStmt := fmt.Sprintf("CREATE SCHEMA %q", qp.DB) _, err := hanaPool.ExecContext(ctx, sqlStmt) return getHanaErrorIfExists(err) } -// DropSchema drops database +// CreateSchemaIfNotExists creates a schema in SAP HANA JSON Document Store. +// +// Returns nil if the schema already exists. +func (hanaPool *Pool) CreateSchemaIfNotExists(ctx context.Context, qp *QueryParams) error { + err := hanaPool.CreateSchema(ctx, qp) + + switch { + case errors.Is(err, ErrSchemaAlreadyExist): + return nil + default: + return err + } +} + +// DropSchema drops database. // -// It returns ErrSchemaNotExist if schema does not exist. -func (hanaPool *Pool) DropSchema(ctx context.Context, db string) error { - sql := fmt.Sprintf("DROP SCHEMA %q CASCADE", db) +// Returns ErrSchemaNotExist if schema does not exist. +func (hanaPool *Pool) DropSchema(ctx context.Context, qp *QueryParams) error { + sql := fmt.Sprintf("DROP SCHEMA %q CASCADE", qp.DB) _, err := hanaPool.ExecContext(ctx, sql) diff --git a/internal/handlers/hana/hanadb/insert.go b/internal/handlers/hana/hanadb/insert.go new file mode 100644 index 000000000000..49b2dfc790dd --- /dev/null +++ b/internal/handlers/hana/hanadb/insert.go @@ -0,0 +1,57 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hanadb + +import ( + "context" + "fmt" + + "github.com/FerretDB/FerretDB/internal/handlers/sjson" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/must" +) + +// CreateSchema inserts a document into a collection in SAP HANA JSON Document Store. +func (hanaPool *Pool) InsertOne(ctx context.Context, qp *QueryParams, doc *types.Document) error { + err := hanaPool.CreateSchemaIfNotExists(ctx, qp) + + switch { + case err == nil: + // Success case + default: + return err + } + + err = hanaPool.CreateCollectionIfNotExists(ctx, qp) + + switch { + case err == nil: + // Success case + default: + return err + } + + return hanaPool.Insert(ctx, qp, doc) +} + +// CreateSchema inserts a document into a collection in SAP HANA JSON Document Store. +func (hanaPool *Pool) Insert(ctx context.Context, qp *QueryParams, doc *types.Document) error { + sqlStmt := fmt.Sprintf("insert into %q.%q values($1)", qp.DB, qp.Collection) + + // sjson.MarshalSingleValue can be used because the Hana insert json format is just a json string + _, err := hanaPool.ExecContext(ctx, sqlStmt, must.NotFail(sjson.MarshalSingleValue(doc))) + + return getHanaErrorIfExists(err) +} diff --git a/internal/handlers/hana/hanadb/query.go b/internal/handlers/hana/hanadb/query.go new file mode 100644 index 000000000000..ea968fa59449 --- /dev/null +++ b/internal/handlers/hana/hanadb/query.go @@ -0,0 +1,21 @@ +// Copyright 2021 FerretDB Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hanadb + +// QueryParams represents options/parameters used for SQL query/statement. +type QueryParams struct { + DB string + Collection string +} diff --git a/internal/handlers/hana/msg_create.go b/internal/handlers/hana/msg_create.go index 7af30795871a..90b44378a239 100644 --- a/internal/handlers/hana/msg_create.go +++ b/internal/handlers/hana/msg_create.go @@ -87,7 +87,12 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, err } - err = dbPool.CreateSchema(ctx, db) + qp := hanadb.QueryParams{ + DB: db, + Collection: collection, + } + + err = dbPool.CreateSchema(ctx, &qp) switch { case err == nil: @@ -101,7 +106,7 @@ func (h *Handler) MsgCreate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, lazyerrors.Error(err) } - err = dbPool.CreateCollection(ctx, db, collection) + err = dbPool.CreateCollection(ctx, &qp) switch { case err == nil: diff --git a/internal/handlers/hana/msg_drop.go b/internal/handlers/hana/msg_drop.go index 1922d9a0300f..98e20115afe2 100644 --- a/internal/handlers/hana/msg_drop.go +++ b/internal/handlers/hana/msg_drop.go @@ -53,7 +53,12 @@ func (h *Handler) MsgDrop(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, er return nil, err } - err = dbPool.DropCollection(ctx, db, collection) + qp := hanadb.QueryParams{ + DB: db, + Collection: collection, + } + + err = dbPool.DropCollection(ctx, &qp) switch { case err == nil: diff --git a/internal/handlers/hana/msg_dropdatabase.go b/internal/handlers/hana/msg_dropdatabase.go index 3bf43bcd628a..10afb7a86242 100644 --- a/internal/handlers/hana/msg_dropdatabase.go +++ b/internal/handlers/hana/msg_dropdatabase.go @@ -45,9 +45,13 @@ func (h *Handler) MsgDropDatabase(ctx context.Context, msg *wire.OpMsg) (*wire.O return nil, err } + qp := hanadb.QueryParams{ + DB: db, + } + res := must.NotFail(types.NewDocument()) - err = dbPool.DropSchema(ctx, db) + err = dbPool.DropSchema(ctx, &qp) switch { case err == nil: diff --git a/internal/handlers/hana/msg_insert.go b/internal/handlers/hana/msg_insert.go index 6c59d8b77e27..4a610fe1f792 100644 --- a/internal/handlers/hana/msg_insert.go +++ b/internal/handlers/hana/msg_insert.go @@ -16,12 +16,134 @@ package hana import ( "context" + "errors" + "fmt" + "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonerrors" + "github.com/FerretDB/FerretDB/internal/handlers/hana/hanadb" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" "github.com/FerretDB/FerretDB/internal/wire" ) // MsgInsert implements HandlerInterface. func (h *Handler) MsgInsert(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, error) { - return nil, notImplemented(must.NotFail(msg.Document()).Command()) + dbPool, err := h.DBPool(ctx) + if err != nil { + return nil, lazyerrors.Error(err) + } + + document, err := msg.Document() + if err != nil { + return nil, lazyerrors.Error(err) + } + + ignoredFields := []string{ + "writeConcern", + } + common.Ignored(document, h.L, ignoredFields...) + + params, err := common.GetInsertParams(document, h.L) + if err != nil { + return nil, err + } + + qp := hanadb.QueryParams{ + DB: params.DB, + Collection: params.Collection, + } + + inserted, insErrors := insertMany(ctx, dbPool, &qp, params.Docs, params.Ordered) + + replyDoc := must.NotFail(types.NewDocument( + "n", inserted, + "ok", float64(1), + )) + + if insErrors.Len() > 0 { + replyDoc = insErrors.Document() + } + + var reply wire.OpMsg + must.NoError(reply.SetSections(wire.OpMsgSection{ + Documents: []*types.Document{replyDoc}, + })) + + return &reply, nil +} + +func insertMany(ctx context.Context, dbPool *hanadb.Pool, qp *hanadb.QueryParams, docs *types.Array, ordered bool) (int32, *commonerrors.WriteErrors) { //nolint:lll // argument list is too long + var inserted int32 + var insErrors commonerrors.WriteErrors + + // TODO: Bulk Insert + // Attempt to insert all the documents in the same request to make insert faster. + /*if err := dbPool.InsertManyDocuments(ctx, qp docs); err == nil { + return int32(docs.Len()), &insErrors + }*/ + + // If the transaction failed, attempt to insert each document separately. + for i := 0; i < docs.Len(); i++ { + doc := must.NotFail(docs.Get(i)) + + err := insertOne(ctx, dbPool, qp, doc.(*types.Document)) + + var we *commonerrors.WriteErrors + + switch { + case err == nil: + inserted++ + continue + case errors.As(err, &we): + insErrors.Merge(we, int32(i)) + default: + insErrors.Append(err, int32(i)) + } + + if ordered { + return inserted, &insErrors + } + } + + return inserted, &insErrors +} + +// insertOne checks if database and collection exist, create them if needed and attempts to insertDocument the given doc. +func insertOne(ctx context.Context, dbPool *hanadb.Pool, qp *hanadb.QueryParams, doc *types.Document) error { + toInsert := doc + + if !toInsert.Has("_id") { + // Make a copy so that original document could be sent to the proxy as it is. + toInsert = doc.DeepCopy() + + toInsert.Set("_id", types.NewObjectID()) + } + + err := dbPool.InsertOne(ctx, qp, toInsert) + + switch { + case err == nil: + return nil + case errors.Is(err, hanadb.ErrInvalidCollectionName), errors.Is(err, hanadb.ErrInvalidDatabaseName): + msg := fmt.Sprintf("Invalid namespace: %s.%s", qp.DB, qp.Collection) + return commonerrors.NewCommandErrorMsg(commonerrors.ErrInvalidNamespace, msg) + + // TODO: set up some sort of metadata table to keep track of '_ids' so we can track duplicates + /*case errors.Is(err, hanzdb.ErrUniqueViolation): + // TODO Extend message for non-_id unique indexes in https://github.com/FerretDB/FerretDB/issues/2045 + idMasrshaled := must.NotFail(json.Marshal(must.NotFail(d.Get("_id")))) + + return commonerrors.NewWriteErrorMsg( + commonerrors.ErrDuplicateKey, + fmt.Sprintf( + `E11000 duplicate key error collection: %s.%s index: _id_ dup key: { _id: %s }`, + qp.DB, qp.Collection, idMasrshaled, + ), + ) + */ + default: + return lazyerrors.Error(err) + } } From ef673cb49579db063b8decd94c2cedad4aa3c6a6 Mon Sep 17 00:00:00 2001 From: Lorant Polya Date: Fri, 2 Jun 2023 10:46:53 -0700 Subject: [PATCH 2/6] Hana Insert Impl Basics linter fixes --- .golangci.yml | 5 ++++- internal/handlers/hana/hanadb/insert.go | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 5a386d279ff6..f7ea8cfc3661 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -169,13 +169,16 @@ issues: path: cmd/envtool text: pgdb - # only `pg` handler and `sqlite` backend can import `sjson` package, no other handlers or backends can do that + # only `pg` handler, `sqlite` backend, and 'hanadb' package can import `sjson` package, no other handlers or backends can do that - linters: [depguard] path: internal/handlers/pg text: sjson - linters: [depguard] path: internal/backends/sqlite test: sjson + - linters: [depguard] + path: internal/handlers/hana/hanadb + test: sjson # only `tigris` handler can import `tigrisdb` package, no other handler can do that - linters: [depguard] diff --git a/internal/handlers/hana/hanadb/insert.go b/internal/handlers/hana/hanadb/insert.go index 49b2dfc790dd..28ec4aba0136 100644 --- a/internal/handlers/hana/hanadb/insert.go +++ b/internal/handlers/hana/hanadb/insert.go @@ -23,7 +23,7 @@ import ( "github.com/FerretDB/FerretDB/internal/util/must" ) -// CreateSchema inserts a document into a collection in SAP HANA JSON Document Store. +// InsertOne inserts a document into a collection in SAP HANA JSON Document Store. func (hanaPool *Pool) InsertOne(ctx context.Context, qp *QueryParams, doc *types.Document) error { err := hanaPool.CreateSchemaIfNotExists(ctx, qp) @@ -43,11 +43,11 @@ func (hanaPool *Pool) InsertOne(ctx context.Context, qp *QueryParams, doc *types return err } - return hanaPool.Insert(ctx, qp, doc) + return hanaPool.insert(ctx, qp, doc) } -// CreateSchema inserts a document into a collection in SAP HANA JSON Document Store. -func (hanaPool *Pool) Insert(ctx context.Context, qp *QueryParams, doc *types.Document) error { +// insert inserts a document into a collection in SAP HANA JSON Document Store. +func (hanaPool *Pool) insert(ctx context.Context, qp *QueryParams, doc *types.Document) error { sqlStmt := fmt.Sprintf("insert into %q.%q values($1)", qp.DB, qp.Collection) // sjson.MarshalSingleValue can be used because the Hana insert json format is just a json string From dca2a74e59449f6d412415ecbe1e4cca22b53d82 Mon Sep 17 00:00:00 2001 From: Lorant Polya Date: Tue, 6 Jun 2023 10:03:15 -0700 Subject: [PATCH 3/6] Update linter --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 6f882caee867..e6ef30dd403b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -41,6 +41,7 @@ linters-settings: - $all - "!**/internal/backends/sqlite/*.go" - "!**/internal/handlers/pg/pgdb/*.go" + - "!**/internal/handlers/hana/hanadb/*.go" deny: - pkg: github.com/FerretDB/FerretDB/internal/handlers/sjson pgdb: From f35752891c154dc3fe64960a47703a9648a62ca6 Mon Sep 17 00:00:00 2001 From: Lorant Polya Date: Thu, 15 Jun 2023 17:07:58 -0700 Subject: [PATCH 4/6] implemented msg_listdatabases --- internal/handlers/hana/hanadb/collections.go | 26 ++++++ internal/handlers/hana/hanadb/databases.go | 92 ++++++++++++++++++- internal/handlers/hana/msg_listdatabases.go | 97 +++++++++++++++++++- 3 files changed, 213 insertions(+), 2 deletions(-) diff --git a/internal/handlers/hana/hanadb/collections.go b/internal/handlers/hana/hanadb/collections.go index 158b60d30521..03147e9bcc3d 100644 --- a/internal/handlers/hana/hanadb/collections.go +++ b/internal/handlers/hana/hanadb/collections.go @@ -18,6 +18,8 @@ import ( "context" "errors" "fmt" + + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" ) // CreateCollection creates a new SAP HANA JSON Document Store collection. @@ -55,3 +57,27 @@ func (hanaPool *Pool) DropCollection(ctx context.Context, qp *QueryParams) error return getHanaErrorIfExists(err) } + +// CollectionSize calculates the size of the given collection. +// +// Returns 0 if schema or collection doesn't exist, otherwise returns its size. +func (hanaPool *Pool) CollectionSize(ctx context.Context, qp *QueryParams) (int64, error) { + sqlStmt := "SELECT TABLE_SIZE FROM M_TABLES WHERE SCHEMA_NAME = $1 AND TABLE_NAME = $2 AND TABLE_TYPE = 'COLLECTION'" + + var size any + if err := hanaPool.QueryRowContext(ctx, sqlStmt, qp.DB, qp.Collection).Scan(&size); err != nil { + return 0, lazyerrors.Error(err) + } + + var collectionSize int64 + switch size := size.(type) { + case int64: + collectionSize = size + case nil: + collectionSize = 0 + default: + return 0, lazyerrors.Errorf("Got wrong type for tableSize. Got: %T", collectionSize) + } + + return collectionSize, nil +} diff --git a/internal/handlers/hana/hanadb/databases.go b/internal/handlers/hana/hanadb/databases.go index 614191a2be84..0f1420157888 100644 --- a/internal/handlers/hana/hanadb/databases.go +++ b/internal/handlers/hana/hanadb/databases.go @@ -18,11 +18,13 @@ import ( "context" "errors" "fmt" + + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" ) // CreateSchema creates a schema in SAP HANA JSON Document Store. // -// Returns ErrSchemaAlreadyExist if schema already exists. +// Returns ErrSchemaAlreadyExist if schema already exists. func (hanaPool *Pool) CreateSchema(ctx context.Context, qp *QueryParams) error { sqlStmt := fmt.Sprintf("CREATE SCHEMA %q", qp.DB) @@ -55,3 +57,91 @@ func (hanaPool *Pool) DropSchema(ctx context.Context, qp *QueryParams) error { return getHanaErrorIfExists(err) } + +// ListSchemas lists all schemas that aren't related to Hana SYS schemas and SYS owner. +func (hanaPool *Pool) ListSchemas(ctx context.Context) ([]string, error) { + const excludeSYS = "%SYS%" + sqlStmt := "SELECT SCHEMA_NAME FROM SCHEMAS WHERE SCHEMA_NAME NOT LIKE $1 AND SCHEMA_OWNER NOT LIKE $2" + + rows, err := hanaPool.QueryContext(ctx, sqlStmt, excludeSYS, excludeSYS) + if err != nil { + return nil, lazyerrors.Error(err) + } + defer rows.Close() + + res := make([]string, 0, 2) + + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + return nil, lazyerrors.Error(err) + } + + res = append(res, name) + } + + if err = rows.Err(); err != nil { + return nil, lazyerrors.Error(err) + } + + return res, nil +} + +// ListCollections lists all collections under a given schema. +// +// Returns an empty array if schema doesn't exist. +func (hanaPool *Pool) ListCollections(ctx context.Context, qp *QueryParams) ([]string, error) { + sqlStmt := "SELECT TABLE_NAME FROM M_TABLES WHERE SCHEMA_NAME = $1 AND TABLE_TYPE = 'COLLECTION'" + + rows, err := hanaPool.QueryContext(ctx, sqlStmt, qp.DB) + if err != nil { + return nil, lazyerrors.Error(err) + } + defer rows.Close() + + res := make([]string, 0, 2) + + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + return nil, lazyerrors.Error(err) + } + + res = append(res, name) + } + + if err = rows.Err(); err != nil { + return nil, lazyerrors.Error(err) + } + + return res, nil +} + +// SchemaSize calculates the total size of collections under schema. +// +// Returns 0 if schema doesn't exist, otherwise returns its size. +func (hanaPool *Pool) SchemaSize(ctx context.Context, qp *QueryParams) (int64, error) { + collections, err := hanaPool.ListCollections(ctx, qp) + if err != nil { + return 0, lazyerrors.Error(err) + } + + qpCopy := QueryParams{ + DB: qp.DB, + } + + var totalSize int64 + + for _, collection := range collections { + qpCopy.Collection = collection + size, err := hanaPool.CollectionSize(ctx, &qpCopy) + + if err != nil { + return 0, lazyerrors.Error(err) + } + + totalSize += size + } + + return totalSize, nil +} diff --git a/internal/handlers/hana/msg_listdatabases.go b/internal/handlers/hana/msg_listdatabases.go index 5cb1a2139521..facd557f1426 100644 --- a/internal/handlers/hana/msg_listdatabases.go +++ b/internal/handlers/hana/msg_listdatabases.go @@ -17,11 +17,106 @@ package hana import ( "context" + "github.com/FerretDB/FerretDB/internal/handlers/common" + "github.com/FerretDB/FerretDB/internal/handlers/commonparams" + "github.com/FerretDB/FerretDB/internal/handlers/hana/hanadb" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/lazyerrors" "github.com/FerretDB/FerretDB/internal/util/must" "github.com/FerretDB/FerretDB/internal/wire" ) // MsgListDatabases implements HandlerInterface. func (h *Handler) MsgListDatabases(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, error) { - return nil, notImplemented(must.NotFail(msg.Document()).Command()) + dbPool, err := h.DBPool(ctx) + if err != nil { + return nil, lazyerrors.Error(err) + } + + document, err := msg.Document() + if err != nil { + return nil, lazyerrors.Error(err) + } + + var filter *types.Document + if filter, err = common.GetOptionalParam(document, "filter", filter); err != nil { + return nil, err + } + + common.Ignored(document, h.L, "comment", "authorizedDatabases") + + databaseNames, err := dbPool.ListSchemas(ctx) + if err != nil { + return nil, err + } + + var nameOnly bool + if v, _ := document.Get("nameOnly"); v != nil { + nameOnly, err = commonparams.GetBoolOptionalParam("nameOnly", v) + if err != nil { + return nil, err + } + } + + var totalSize int64 + databases := types.MakeArray(len(databaseNames)) + qp := hanadb.QueryParams{} + + for _, databaseName := range databaseNames { + qp.DB = databaseName + + size, err := dbPool.SchemaSize(ctx, &qp) + if err != nil { + return nil, lazyerrors.Error(err) + } + + totalSize += size + + d := must.NotFail(types.NewDocument( + "name", databaseName, + "sizeOnDisk", size, + "empty", size == 0, + )) + + matches, err := common.FilterDocument(d, filter) + if err != nil { + return nil, err + } + + if !matches { + continue + } + + if nameOnly { + d = must.NotFail(types.NewDocument( + "name", databaseName, + )) + } + + databases.Append(d) + } + + if nameOnly { + var reply wire.OpMsg + must.NoError(reply.SetSections(wire.OpMsgSection{ + Documents: []*types.Document{must.NotFail(types.NewDocument( + "databases", databases, + "ok", float64(1), + ))}, + })) + + return &reply, nil + } + + var reply wire.OpMsg + must.NoError(reply.SetSections(wire.OpMsgSection{ + Documents: []*types.Document{must.NotFail(types.NewDocument( + "databases", databases, + "totalSize", totalSize, + "totalSizeMb", totalSize/1024/1024, + "ok", float64(1), + ))}, + })) + + return &reply, nil } From e6441d999ed41dd107aa7d0205c63b184be01765 Mon Sep 17 00:00:00 2001 From: Lorant Polya Date: Tue, 20 Jun 2023 08:34:18 -0700 Subject: [PATCH 5/6] fixed linter issue --- internal/handlers/hana/hanadb/databases.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/hana/hanadb/databases.go b/internal/handlers/hana/hanadb/databases.go index 0f1420157888..80cf538d407c 100644 --- a/internal/handlers/hana/hanadb/databases.go +++ b/internal/handlers/hana/hanadb/databases.go @@ -134,8 +134,8 @@ func (hanaPool *Pool) SchemaSize(ctx context.Context, qp *QueryParams) (int64, e for _, collection := range collections { qpCopy.Collection = collection - size, err := hanaPool.CollectionSize(ctx, &qpCopy) + size, err := hanaPool.CollectionSize(ctx, &qpCopy) if err != nil { return 0, lazyerrors.Error(err) } From 328bbfedbd02f5f54d804e8fe178f8d7104d9b45 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 21 Jun 2023 17:14:10 +0400 Subject: [PATCH 6/6] Shard Hana tests too --- .github/workflows/go-trust.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/go-trust.yml b/.github/workflows/go-trust.yml index 5dc8c72ebc12..1b21ca25de5a 100644 --- a/.github/workflows/go-trust.yml +++ b/.github/workflows/go-trust.yml @@ -32,10 +32,16 @@ env: jobs: integration: - name: Integration ${{ matrix.name }} + name: Integration ${{ matrix.name }} ${{ matrix.shard_index }}/${{ matrix.shard_total }} runs-on: group: paid - timeout-minutes: 30 + timeout-minutes: 20 + + # Do not run this job in parallel for any PR change or branch/tag push + # to save some resources. + concurrency: + group: ${{ github.workflow }}-${{ matrix.name }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true if: > github.event_name != 'pull_request_target' || @@ -48,7 +54,9 @@ jobs: fail-fast: false matrix: include: - - { name: "Hana", task: "hana" } + - { name: "Hana", task: "hana", shard_index: 1, shard_total: 3 } + - { name: "Hana", task: "hana", shard_index: 2, shard_total: 3 } + - { name: "Hana", task: "hana", shard_index: 3, shard_total: 3 } steps: - name: Checkout code @@ -90,8 +98,8 @@ jobs: - name: Wait for and setup environment run: bin/task env-setup - - name: Run ${{ matrix.task }} tests - run: bin/task test-integration-${{ matrix.task }} + - name: Run ${{ matrix.name }} tests (${{ matrix.shard_index }}/${{ matrix.shard_total }}) + run: bin/task test-integration-${{ matrix.task }} SHARD_INDEX=${{ matrix.shard_index }} SHARD_TOTAL=${{ matrix.shard_total }} env: GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }} FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }} @@ -108,7 +116,7 @@ jobs: with: token: 22159d7c-856d-4fe9-8fdb-5d9ecff35514 files: ./integration/integration-${{ matrix.task }}.txt - flags: integration,${{ matrix.task }} + flags: integration,${{ matrix.task }},shard-${{ matrix.shard_index }} fail_ci_if_error: true verbose: true