From 6d00a16a20a6860da8e8fcd22851160f8bad46e2 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 12:48:04 +0900 Subject: [PATCH 01/27] include capped collection details in response --- integration/commands_administration_test.go | 13 +++++++++++-- internal/backends/collection.go | 13 ++++++++----- internal/handlers/sqlite/msg_collstats.go | 14 +++++++++++++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index a05cd632645e..85049314392e 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -698,8 +698,6 @@ func TestCommandsAdministrationCollStatsEmpty(t *testing.T) { } func TestCommandsAdministrationCollStats(t *testing.T) { - setup.SkipForNewPg(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) @@ -729,6 +727,17 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) assert.InDelta(t, 12_000, must.NotFail(doc.Get("totalIndexSize")), 11_000) assert.InDelta(t, 32_000, must.NotFail(doc.Get("totalSize")), 30_000) + + // old PG handler does not include capped collection details in response + setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") + capped, _ := doc.Get("capped") + assert.Equal(t, false, capped) + + max, _ := doc.Get("max") + assert.Nil(t, max) + + maxSize, _ := doc.Get("maxSize") + assert.Nil(t, maxSize) } func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 63efcd981980..b68b9f15db74 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -226,15 +226,18 @@ type CollectionStatsParams struct { // CollectionStatsResult represents the results of Collection.Stats method. // +// CappedSize and CappedDocuments are zero for not capped collection. // CountObjects is an estimate of the number of documents. // // TODO https://github.com/FerretDB/FerretDB/issues/2447 type CollectionStatsResult struct { - CountObjects int64 - CountIndexes int64 - SizeTotal int64 - SizeIndexes int64 - SizeCollection int64 + CappedSize int64 + CappedDocuments int64 + CountObjects int64 + CountIndexes int64 + SizeTotal int64 + SizeIndexes int64 + SizeCollection int64 } // Stats returns statistics about the collection. diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 221581ecc7a3..5ec010cb2b08 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -97,6 +97,8 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) } + capped := stats.CappedSize != int64(0) + // MongoDB uses "numbers" that could be int32 or int64, // FerretDB always returns int64 for simplicity. pairs = append(pairs, @@ -105,7 +107,17 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs "totalIndexSize", stats.SizeIndexes/scale, "totalSize", stats.SizeTotal/scale, "scaleFactor", int32(scale), - "capped", false, // TODO https://github.com/FerretDB/FerretDB/issues/3458 + "capped", capped, + ) + + if capped { + pairs = append(pairs, + "max", stats.CappedDocuments, + "maxSize", stats.CappedSize, + ) + } + + pairs = append(pairs, "ok", float64(1), ) From 9902948e89aae42e5f1f542861f01850aa0691e7 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 14:15:24 +0900 Subject: [PATCH 02/27] unskip collstats for new pg backend --- integration/commands_administration_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 85049314392e..dde256ec94f2 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -741,8 +741,6 @@ func TestCommandsAdministrationCollStats(t *testing.T) { } func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { - setup.SkipForNewPg(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) From 1c7aafe76ae76a7faa8fc16ba3f80c6c80d7ccaa Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 14:30:43 +0900 Subject: [PATCH 03/27] unskip datasize test --- integration/commands_administration_test.go | 2 -- internal/handlers/sqlite/msg_collstats.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index dde256ec94f2..a2a0965bbef2 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -788,8 +788,6 @@ func TestCommandsAdministrationCollStatsCount(t *testing.T) { } func TestCommandsAdministrationDataSize(t *testing.T) { - setup.SkipForNewPg(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() t.Run("Existing", func(t *testing.T) { diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 5ec010cb2b08..561e5e3e8247 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -113,7 +113,7 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs if capped { pairs = append(pairs, "max", stats.CappedDocuments, - "maxSize", stats.CappedSize, + "maxSize", stats.CappedSize/scale, ) } From e4de4c34e624eadd2e4175714a8e31ba3caa8613 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 15:11:15 +0900 Subject: [PATCH 04/27] add compat test for testing capped collection in collstats --- .../commands_administration_compat_test.go | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index dcc7fbbaa11f..d8c1e6095a2d 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -18,13 +18,16 @@ import ( "math" "testing" + "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" "github.com/FerretDB/FerretDB/integration/setup" "github.com/FerretDB/FerretDB/integration/shareddata" "github.com/FerretDB/FerretDB/internal/util/must" + "github.com/FerretDB/FerretDB/internal/util/testutil" ) func TestCommandsAdministrationCompatCollStatsWithScale(t *testing.T) { @@ -97,6 +100,68 @@ func TestCommandsAdministrationCompatCollStatsWithScale(t *testing.T) { } } +func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { + t.Skip("https://github.com/FerretDB/FerretDB/issues/3458") + + t.Parallel() + + s := setup.SetupCompatWithOpts(t, &setup.SetupCompatOpts{ + Providers: []shareddata.Provider{}, + AddNonExistentCollection: true, + }) + + ctx, targetCollection, compatCollection := s.Ctx, s.TargetCollections[0], s.CompatCollections[0] + + for name, tc := range map[string]struct { //nolint:vet // for readability + opts *options.CreateCollectionOptions + }{ + "Size": { + opts: &options.CreateCollectionOptions{ + Capped: pointer.ToBool(true), + SizeInBytes: pointer.ToInt64(1000), + MaxDocuments: pointer.ToInt64(10), + }, + }, + "MaxDocuments": { + opts: &options.CreateCollectionOptions{ + Capped: pointer.ToBool(true), + SizeInBytes: pointer.ToInt64(1000), + MaxDocuments: pointer.ToInt64(10), + }, + }, + } { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + cName := testutil.CollectionName(t) + name + targetErr := targetCollection.Database().CreateCollection(ctx, cName, tc.opts) + require.NoError(t, targetErr) + + compatErr := compatCollection.Database().CreateCollection(ctx, cName, tc.opts) + require.NoError(t, compatErr) + + require.Equal(t, compatCollection.Name(), targetCollection.Name()) + command := bson.D{{"collStats", targetCollection.Name()}} + + var targetRes bson.D + targetErr = targetCollection.Database().RunCommand(ctx, command).Decode(&targetRes) + require.NoError(t, targetErr) + + var compatRes bson.D + targetErr = compatCollection.Database().RunCommand(ctx, command).Decode(&targetRes) + require.NoError(t, targetErr) + + targetDoc := ConvertDocument(t, targetRes) + compatDoc := ConvertDocument(t, compatRes) + + assert.Equal(t, must.NotFail(compatDoc.Get("capped")), must.NotFail(targetDoc.Get("capped"))) + assert.Equal(t, must.NotFail(compatDoc.Get("max")), must.NotFail(targetDoc.Get("max"))) + assert.Equal(t, must.NotFail(compatDoc.Get("maxSize")), must.NotFail(targetDoc.Get("maxSize"))) + }) + } +} + func TestCommandsAdministrationCompatDBStatsWithScale(t *testing.T) { t.Parallel() From 6667680b3135beab0903ecf08460751dd063a370 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 15:28:15 +0900 Subject: [PATCH 05/27] increase delta --- integration/commands_administration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index a2a0965bbef2..c3352fbd4a81 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -804,7 +804,9 @@ func TestCommandsAdministrationDataSize(t *testing.T) { assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) assert.InDelta(t, 24_576, must.NotFail(doc.Get("size")), 24_576) assert.InDelta(t, 4, must.NotFail(doc.Get("numObjects")), 4) // TODO https://github.com/FerretDB/FerretDB/issues/727 - assert.InDelta(t, 200, must.NotFail(doc.Get("millis")), 200) + // reduce delta to 200 once ANALYZE is called less + // TODO https://github.com/FerretDB/FerretDB/issues/3518 + assert.InDelta(t, 200, must.NotFail(doc.Get("millis")), 300) }) t.Run("NonExistent", func(t *testing.T) { From 0854b7eb205bb99b16a850e45c1c747dcb2e6a3c Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 17:54:43 +0900 Subject: [PATCH 06/27] update how collstats results are compared --- integration/commands_administration_test.go | 99 +++++++++++++++++---- integration/shareddata/composites.go | 2 + internal/handlers/sqlite/msg_collstats.go | 16 ++-- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index c3352fbd4a81..30e34eb3c389 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -719,14 +719,52 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.Equal(t, int32(1), must.NotFail(doc.Get("scaleFactor"))) assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) - // Values are returned as "numbers" that could be int32 or int64. - // FerretDB always returns int64 for simplicity. - assert.InDelta(t, 40_000, must.NotFail(doc.Get("size")), 39_900) - assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) - assert.InDelta(t, 40_000, must.NotFail(doc.Get("storageSize")), 39_900) - assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) - assert.InDelta(t, 12_000, must.NotFail(doc.Get("totalIndexSize")), 11_000) - assert.InDelta(t, 32_000, must.NotFail(doc.Get("totalSize")), 30_000) + type defaultSize struct { + collectionSize int32 + storageSize int32 + indexSize int32 + } + + // The expected sizes are vastly different for each database due to + // how much storage is allocated for collection and indexes. + // MongoDB differentiates the size used by collection and the storage allocated + // for collection, FerretDB does not differentiate them. + // Hence, the expectation is defined for each backend. + var expectedSizes defaultSize + switch { + case setup.IsPostgres(t): + expectedSizes = defaultSize{ + collectionSize: 16384, + storageSize: 16384, + indexSize: 16384, + } + case setup.IsSQLite(t): + expectedSizes = defaultSize{ + collectionSize: 4096, + storageSize: 4096, + indexSize: 4096, + } + case setup.IsMongoDB(t): + expectedSizes = defaultSize{ + collectionSize: 298, + storageSize: 4096, + indexSize: 4096, + } + default: + t.Fatalf("expected size is not defined for the backend") + } + + assert.NotZero(t, must.NotFail(doc.Get("size"))) + assert.LessOrEqual(t, must.NotFail(doc.Get("size")), expectedSizes.collectionSize) + + expectedAvgObjSize := expectedSizes.collectionSize / must.NotFail(doc.Get("count")).(int32) + assert.NotZero(t, expectedAvgObjSize) + assert.InDelta(t, expectedAvgObjSize, must.NotFail(doc.Get("avgObjSize")), float64(expectedAvgObjSize)) + + assert.Equal(t, expectedSizes.storageSize, must.NotFail(doc.Get("storageSize"))) + assert.Equal(t, int32(1), must.NotFail(doc.Get("nindexes"))) + assert.Equal(t, expectedSizes.indexSize, must.NotFail(doc.Get("totalIndexSize"))) + assert.Equal(t, expectedSizes.storageSize+expectedSizes.indexSize, must.NotFail(doc.Get("totalSize"))) // old PG handler does not include capped collection details in response setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") @@ -745,25 +783,48 @@ func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) - var actual bson.D - command := bson.D{{"collStats", collection.Name()}, {"scale", float64(1_000)}} - err := collection.Database().RunCommand(ctx, command).Decode(&actual) + scale := 1_000 + var res bson.D + err := collection.Database().RunCommand(ctx, bson.D{ + {"collStats", collection.Name()}, + {"scale", float64(scale)}, + }).Decode(&res) require.NoError(t, err) - // Set better expected results. - // TODO https://github.com/FerretDB/FerretDB/issues/1771 - doc := ConvertDocument(t, actual) + var resWithoutScale bson.D + err = collection.Database().RunCommand(ctx, bson.D{ + {"collStats", collection.Name()}, + }).Decode(&resWithoutScale) + require.NoError(t, err) + + doc := ConvertDocument(t, res) + docWithoutScale := ConvertDocument(t, resWithoutScale) assert.Equal(t, collection.Database().Name()+"."+collection.Name(), must.NotFail(doc.Get("ns"))) assert.EqualValues(t, 6, must.NotFail(doc.Get("count"))) // Number of documents in DocumentsStrings assert.Equal(t, int32(1000), must.NotFail(doc.Get("scaleFactor"))) assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) - assert.InDelta(t, 16, must.NotFail(doc.Get("size")), 16) - assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) - assert.InDelta(t, 24, must.NotFail(doc.Get("storageSize")), 24) + size := must.NotFail(doc.Get("size")) + sizeWithoutScale := must.NotFail(docWithoutScale.Get("size")).(int32) + assert.EqualValues(t, sizeWithoutScale/int32(scale), size) + + avgObjSize := must.NotFail(doc.Get("avgObjSize")) + avgObjSizeWithoutScale := must.NotFail(docWithoutScale.Get("avgObjSize")).(int32) + assert.EqualValues(t, avgObjSizeWithoutScale, avgObjSize) + + storageSize := must.NotFail(doc.Get("storageSize")) + storageSizeWithoutScale := must.NotFail(docWithoutScale.Get("storageSize")).(int32) + assert.EqualValues(t, storageSizeWithoutScale/int32(scale), storageSize) + assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) - assert.InDelta(t, 8, must.NotFail(doc.Get("totalIndexSize")), 8) - assert.InDelta(t, 24, must.NotFail(doc.Get("totalSize")), 24) + + totalIndexSize := must.NotFail(doc.Get("totalIndexSize")) + totalIndexSizeWithoutScale := must.NotFail(docWithoutScale.Get("totalIndexSize")).(int32) + assert.EqualValues(t, totalIndexSizeWithoutScale/int32(scale), totalIndexSize) + + totalSize := must.NotFail(doc.Get("totalSize")) + totalSizeWithoutScale := must.NotFail(docWithoutScale.Get("totalSize")).(int32) + assert.EqualValues(t, totalSizeWithoutScale/int32(scale), totalSize) } // TestCommandsAdministrationCollStatsCount adds large number of documents and checks diff --git a/integration/shareddata/composites.go b/integration/shareddata/composites.go index e28c93854363..f3793c27c352 100644 --- a/integration/shareddata/composites.go +++ b/integration/shareddata/composites.go @@ -139,6 +139,8 @@ var DocumentsDoubles = &Values[string]{ } // DocumentsStrings contains documents with string values for tests. +// It is used by CollStats for testing stored object and index sizes, +// modifying this may require updating CollStats tests. var DocumentsStrings = &Values[string]{ name: "DocumentsStrings", data: map[string]any{ diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 561e5e3e8247..64c7003e9d5c 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -88,24 +88,22 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs := []any{ "ns", dbName + "." + collection, - "size", stats.SizeCollection / scale, - "count", stats.CountObjects, + "size", int32(stats.SizeCollection / scale), + "count", int32(stats.CountObjects), } // If there are objects in the collection, calculate the average object size. if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) + pairs = append(pairs, "avgObjSize", int32(stats.SizeCollection/stats.CountObjects)) } capped := stats.CappedSize != int64(0) - // MongoDB uses "numbers" that could be int32 or int64, - // FerretDB always returns int64 for simplicity. pairs = append(pairs, - "storageSize", stats.SizeCollection/scale, - "nindexes", stats.CountIndexes, - "totalIndexSize", stats.SizeIndexes/scale, - "totalSize", stats.SizeTotal/scale, + "storageSize", int32(stats.SizeCollection/scale), + "nindexes", int32(stats.CountIndexes), + "totalIndexSize", int32(stats.SizeIndexes/scale), + "totalSize", int32(stats.SizeTotal/scale), "scaleFactor", int32(scale), "capped", capped, ) From a31f3e44e866ec44b02e0fae5e2283aeb4419585 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 11:35:53 +0900 Subject: [PATCH 07/27] update expectation for collstats --- integration/commands_administration_test.go | 11 ++++------- integration/shareddata/composites.go | 3 ++- internal/handlers/sqlite/msg_aggregate.go | 5 ++--- internal/handlers/sqlite/msg_collstats.go | 4 ++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 30e34eb3c389..765602c7db8b 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -709,11 +709,6 @@ func TestCommandsAdministrationCollStats(t *testing.T) { doc := ConvertDocument(t, actual) - // Values are returned as "numbers" that could be int32 or int64. - // FerretDB always returns int64 for simplicity. - // - // Set better expected results. - // TODO https://github.com/FerretDB/FerretDB/issues/1771 assert.Equal(t, collection.Database().Name()+"."+collection.Name(), must.NotFail(doc.Get("ns"))) assert.EqualValues(t, 6, must.NotFail(doc.Get("count"))) // // Number of documents in DocumentsStrings assert.Equal(t, int32(1), must.NotFail(doc.Get("scaleFactor"))) @@ -725,8 +720,8 @@ func TestCommandsAdministrationCollStats(t *testing.T) { indexSize int32 } - // The expected sizes are vastly different for each database due to - // how much storage is allocated for collection and indexes. + // The expected sizes are vastly different for each database due to how much storage + // is allocated for collection and indexes. // MongoDB differentiates the size used by collection and the storage allocated // for collection, FerretDB does not differentiate them. // Hence, the expectation is defined for each backend. @@ -778,6 +773,8 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.Nil(t, maxSize) } +// TestCommandsAdministrationCollStatsWithScale asserts scale is set appropriately by +// calling collStats with and without scale and compares the response. func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { t.Parallel() diff --git a/integration/shareddata/composites.go b/integration/shareddata/composites.go index f3793c27c352..7ef1a6ebc3aa 100644 --- a/integration/shareddata/composites.go +++ b/integration/shareddata/composites.go @@ -139,7 +139,8 @@ var DocumentsDoubles = &Values[string]{ } // DocumentsStrings contains documents with string values for tests. -// It is used by CollStats for testing stored object and index sizes, +// +// It is used by CollStats for testing stored objects and index sizes, // modifying this may require updating CollStats tests. var DocumentsStrings = &Values[string]{ name: "DocumentsStrings", diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 59b4ca5b5b84..d454942a2b73 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -409,14 +409,13 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, "freeStorageSize", int64(0), // TODO https://github.com/FerretDB/FerretDB/issues/2342 - "capped", false, // TODO https://github.com/FerretDB/FerretDB/issues/2342 - "wiredTiger", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 + "capped", collStats.CappedSize != int64(0), "nindexes", collStats.CountIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "totalIndexSize", collStats.SizeIndexes, "totalSize", collStats.SizeTotal, - "indexSizes", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 + "indexSizes", collStats.SizeIndexes, )), ) } diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 64c7003e9d5c..e94737ff5e60 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -110,8 +110,8 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs if capped { pairs = append(pairs, - "max", stats.CappedDocuments, - "maxSize", stats.CappedSize/scale, + "max", int32(stats.CappedDocuments), + "maxSize", int32(stats.CappedSize/scale), ) } From 98bd5e5fbc8c0aaa2764b92bda13f31caab8ec27 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 12:10:30 +0900 Subject: [PATCH 08/27] skip test for old pg --- integration/commands_administration_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 765602c7db8b..986d6aa17adb 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -698,6 +698,9 @@ func TestCommandsAdministrationCollStatsEmpty(t *testing.T) { } func TestCommandsAdministrationCollStats(t *testing.T) { + // old PG handler returns sizes in int64 instead of int32 + setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") + t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) @@ -776,6 +779,9 @@ func TestCommandsAdministrationCollStats(t *testing.T) { // TestCommandsAdministrationCollStatsWithScale asserts scale is set appropriately by // calling collStats with and without scale and compares the response. func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { + // old PG handler returns sizes in int64 instead of int32 + setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") + t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) From 68bb0a22bac344a31eab8f90da0cd22d4a771709 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 12:33:00 +0900 Subject: [PATCH 09/27] Revert "update expectation for collstats" This reverts commit a31f3e44e866ec44b02e0fae5e2283aeb4419585. --- integration/commands_administration_test.go | 11 +++++++---- integration/shareddata/composites.go | 3 +-- internal/handlers/sqlite/msg_aggregate.go | 5 +++-- internal/handlers/sqlite/msg_collstats.go | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 986d6aa17adb..92d9029a8f98 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -712,6 +712,11 @@ func TestCommandsAdministrationCollStats(t *testing.T) { doc := ConvertDocument(t, actual) + // Values are returned as "numbers" that could be int32 or int64. + // FerretDB always returns int64 for simplicity. + // + // Set better expected results. + // TODO https://github.com/FerretDB/FerretDB/issues/1771 assert.Equal(t, collection.Database().Name()+"."+collection.Name(), must.NotFail(doc.Get("ns"))) assert.EqualValues(t, 6, must.NotFail(doc.Get("count"))) // // Number of documents in DocumentsStrings assert.Equal(t, int32(1), must.NotFail(doc.Get("scaleFactor"))) @@ -723,8 +728,8 @@ func TestCommandsAdministrationCollStats(t *testing.T) { indexSize int32 } - // The expected sizes are vastly different for each database due to how much storage - // is allocated for collection and indexes. + // The expected sizes are vastly different for each database due to + // how much storage is allocated for collection and indexes. // MongoDB differentiates the size used by collection and the storage allocated // for collection, FerretDB does not differentiate them. // Hence, the expectation is defined for each backend. @@ -776,8 +781,6 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.Nil(t, maxSize) } -// TestCommandsAdministrationCollStatsWithScale asserts scale is set appropriately by -// calling collStats with and without scale and compares the response. func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { // old PG handler returns sizes in int64 instead of int32 setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") diff --git a/integration/shareddata/composites.go b/integration/shareddata/composites.go index 7ef1a6ebc3aa..f3793c27c352 100644 --- a/integration/shareddata/composites.go +++ b/integration/shareddata/composites.go @@ -139,8 +139,7 @@ var DocumentsDoubles = &Values[string]{ } // DocumentsStrings contains documents with string values for tests. -// -// It is used by CollStats for testing stored objects and index sizes, +// It is used by CollStats for testing stored object and index sizes, // modifying this may require updating CollStats tests. var DocumentsStrings = &Values[string]{ name: "DocumentsStrings", diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index d454942a2b73..59b4ca5b5b84 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -409,13 +409,14 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, "freeStorageSize", int64(0), // TODO https://github.com/FerretDB/FerretDB/issues/2342 - "capped", collStats.CappedSize != int64(0), + "capped", false, // TODO https://github.com/FerretDB/FerretDB/issues/2342 + "wiredTiger", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "nindexes", collStats.CountIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "totalIndexSize", collStats.SizeIndexes, "totalSize", collStats.SizeTotal, - "indexSizes", collStats.SizeIndexes, + "indexSizes", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 )), ) } diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index e94737ff5e60..64c7003e9d5c 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -110,8 +110,8 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs if capped { pairs = append(pairs, - "max", int32(stats.CappedDocuments), - "maxSize", int32(stats.CappedSize/scale), + "max", stats.CappedDocuments, + "maxSize", stats.CappedSize/scale, ) } From 47611b43b469e58e45d27bbdbb06898340b660f2 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 12:33:17 +0900 Subject: [PATCH 10/27] Revert "skip test for old pg" This reverts commit 98bd5e5fbc8c0aaa2764b92bda13f31caab8ec27. --- integration/commands_administration_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 92d9029a8f98..30e34eb3c389 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -698,9 +698,6 @@ func TestCommandsAdministrationCollStatsEmpty(t *testing.T) { } func TestCommandsAdministrationCollStats(t *testing.T) { - // old PG handler returns sizes in int64 instead of int32 - setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) @@ -782,9 +779,6 @@ func TestCommandsAdministrationCollStats(t *testing.T) { } func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { - // old PG handler returns sizes in int64 instead of int32 - setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) From 176c2540e059ef3c327d458c038f4e3aee2e0d6a Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 12:33:28 +0900 Subject: [PATCH 11/27] Revert "update how collstats results are compared" This reverts commit 0854b7eb205bb99b16a850e45c1c747dcb2e6a3c. --- integration/commands_administration_test.go | 99 ++++----------------- integration/shareddata/composites.go | 2 - internal/handlers/sqlite/msg_collstats.go | 16 ++-- 3 files changed, 28 insertions(+), 89 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 30e34eb3c389..c3352fbd4a81 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -719,52 +719,14 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.Equal(t, int32(1), must.NotFail(doc.Get("scaleFactor"))) assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) - type defaultSize struct { - collectionSize int32 - storageSize int32 - indexSize int32 - } - - // The expected sizes are vastly different for each database due to - // how much storage is allocated for collection and indexes. - // MongoDB differentiates the size used by collection and the storage allocated - // for collection, FerretDB does not differentiate them. - // Hence, the expectation is defined for each backend. - var expectedSizes defaultSize - switch { - case setup.IsPostgres(t): - expectedSizes = defaultSize{ - collectionSize: 16384, - storageSize: 16384, - indexSize: 16384, - } - case setup.IsSQLite(t): - expectedSizes = defaultSize{ - collectionSize: 4096, - storageSize: 4096, - indexSize: 4096, - } - case setup.IsMongoDB(t): - expectedSizes = defaultSize{ - collectionSize: 298, - storageSize: 4096, - indexSize: 4096, - } - default: - t.Fatalf("expected size is not defined for the backend") - } - - assert.NotZero(t, must.NotFail(doc.Get("size"))) - assert.LessOrEqual(t, must.NotFail(doc.Get("size")), expectedSizes.collectionSize) - - expectedAvgObjSize := expectedSizes.collectionSize / must.NotFail(doc.Get("count")).(int32) - assert.NotZero(t, expectedAvgObjSize) - assert.InDelta(t, expectedAvgObjSize, must.NotFail(doc.Get("avgObjSize")), float64(expectedAvgObjSize)) - - assert.Equal(t, expectedSizes.storageSize, must.NotFail(doc.Get("storageSize"))) - assert.Equal(t, int32(1), must.NotFail(doc.Get("nindexes"))) - assert.Equal(t, expectedSizes.indexSize, must.NotFail(doc.Get("totalIndexSize"))) - assert.Equal(t, expectedSizes.storageSize+expectedSizes.indexSize, must.NotFail(doc.Get("totalSize"))) + // Values are returned as "numbers" that could be int32 or int64. + // FerretDB always returns int64 for simplicity. + assert.InDelta(t, 40_000, must.NotFail(doc.Get("size")), 39_900) + assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) + assert.InDelta(t, 40_000, must.NotFail(doc.Get("storageSize")), 39_900) + assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) + assert.InDelta(t, 12_000, must.NotFail(doc.Get("totalIndexSize")), 11_000) + assert.InDelta(t, 32_000, must.NotFail(doc.Get("totalSize")), 30_000) // old PG handler does not include capped collection details in response setup.SkipForOldPg(t, "https://github.com/FerretDB/FerretDB/issues/3435") @@ -783,48 +745,25 @@ func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) - scale := 1_000 - var res bson.D - err := collection.Database().RunCommand(ctx, bson.D{ - {"collStats", collection.Name()}, - {"scale", float64(scale)}, - }).Decode(&res) - require.NoError(t, err) - - var resWithoutScale bson.D - err = collection.Database().RunCommand(ctx, bson.D{ - {"collStats", collection.Name()}, - }).Decode(&resWithoutScale) + var actual bson.D + command := bson.D{{"collStats", collection.Name()}, {"scale", float64(1_000)}} + err := collection.Database().RunCommand(ctx, command).Decode(&actual) require.NoError(t, err) - doc := ConvertDocument(t, res) - docWithoutScale := ConvertDocument(t, resWithoutScale) + // Set better expected results. + // TODO https://github.com/FerretDB/FerretDB/issues/1771 + doc := ConvertDocument(t, actual) assert.Equal(t, collection.Database().Name()+"."+collection.Name(), must.NotFail(doc.Get("ns"))) assert.EqualValues(t, 6, must.NotFail(doc.Get("count"))) // Number of documents in DocumentsStrings assert.Equal(t, int32(1000), must.NotFail(doc.Get("scaleFactor"))) assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) - size := must.NotFail(doc.Get("size")) - sizeWithoutScale := must.NotFail(docWithoutScale.Get("size")).(int32) - assert.EqualValues(t, sizeWithoutScale/int32(scale), size) - - avgObjSize := must.NotFail(doc.Get("avgObjSize")) - avgObjSizeWithoutScale := must.NotFail(docWithoutScale.Get("avgObjSize")).(int32) - assert.EqualValues(t, avgObjSizeWithoutScale, avgObjSize) - - storageSize := must.NotFail(doc.Get("storageSize")) - storageSizeWithoutScale := must.NotFail(docWithoutScale.Get("storageSize")).(int32) - assert.EqualValues(t, storageSizeWithoutScale/int32(scale), storageSize) - + assert.InDelta(t, 16, must.NotFail(doc.Get("size")), 16) + assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) + assert.InDelta(t, 24, must.NotFail(doc.Get("storageSize")), 24) assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) - - totalIndexSize := must.NotFail(doc.Get("totalIndexSize")) - totalIndexSizeWithoutScale := must.NotFail(docWithoutScale.Get("totalIndexSize")).(int32) - assert.EqualValues(t, totalIndexSizeWithoutScale/int32(scale), totalIndexSize) - - totalSize := must.NotFail(doc.Get("totalSize")) - totalSizeWithoutScale := must.NotFail(docWithoutScale.Get("totalSize")).(int32) - assert.EqualValues(t, totalSizeWithoutScale/int32(scale), totalSize) + assert.InDelta(t, 8, must.NotFail(doc.Get("totalIndexSize")), 8) + assert.InDelta(t, 24, must.NotFail(doc.Get("totalSize")), 24) } // TestCommandsAdministrationCollStatsCount adds large number of documents and checks diff --git a/integration/shareddata/composites.go b/integration/shareddata/composites.go index f3793c27c352..e28c93854363 100644 --- a/integration/shareddata/composites.go +++ b/integration/shareddata/composites.go @@ -139,8 +139,6 @@ var DocumentsDoubles = &Values[string]{ } // DocumentsStrings contains documents with string values for tests. -// It is used by CollStats for testing stored object and index sizes, -// modifying this may require updating CollStats tests. var DocumentsStrings = &Values[string]{ name: "DocumentsStrings", data: map[string]any{ diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 64c7003e9d5c..561e5e3e8247 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -88,22 +88,24 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs := []any{ "ns", dbName + "." + collection, - "size", int32(stats.SizeCollection / scale), - "count", int32(stats.CountObjects), + "size", stats.SizeCollection / scale, + "count", stats.CountObjects, } // If there are objects in the collection, calculate the average object size. if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", int32(stats.SizeCollection/stats.CountObjects)) + pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) } capped := stats.CappedSize != int64(0) + // MongoDB uses "numbers" that could be int32 or int64, + // FerretDB always returns int64 for simplicity. pairs = append(pairs, - "storageSize", int32(stats.SizeCollection/scale), - "nindexes", int32(stats.CountIndexes), - "totalIndexSize", int32(stats.SizeIndexes/scale), - "totalSize", int32(stats.SizeTotal/scale), + "storageSize", stats.SizeCollection/scale, + "nindexes", stats.CountIndexes, + "totalIndexSize", stats.SizeIndexes/scale, + "totalSize", stats.SizeTotal/scale, "scaleFactor", int32(scale), "capped", capped, ) From e70f2df71296d70bc076b9cc3c0fe684ad6659f8 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Wed, 11 Oct 2023 19:12:09 +0900 Subject: [PATCH 12/27] add todo --- .../commands_administration_compat_test.go | 2 +- internal/backends/collection.go | 2 -- internal/backends/postgresql/collection.go | 35 ++++++++++++++++--- internal/backends/sqlite/collection.go | 35 ++++++++++++++++--- internal/handlers/sqlite/msg_aggregate.go | 8 ++--- internal/handlers/sqlite/msg_collstats.go | 3 ++ internal/handlers/sqlite/msg_dbstats.go | 3 ++ 7 files changed, 71 insertions(+), 17 deletions(-) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index d8c1e6095a2d..c9d3d8fe9b7b 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -101,7 +101,7 @@ func TestCommandsAdministrationCompatCollStatsWithScale(t *testing.T) { } func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { - t.Skip("https://github.com/FerretDB/FerretDB/issues/3458") + t.Skip("https://github.com/FerretDB/FerretDB/issues/2447") t.Parallel() diff --git a/internal/backends/collection.go b/internal/backends/collection.go index b68b9f15db74..cab24acdbadf 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -228,8 +228,6 @@ type CollectionStatsParams struct { // // CappedSize and CappedDocuments are zero for not capped collection. // CountObjects is an estimate of the number of documents. -// -// TODO https://github.com/FerretDB/FerretDB/issues/2447 type CollectionStatsResult struct { CappedSize int64 CappedDocuments int64 diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index b585f197c5a5..49d1eaf81c70 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -336,17 +336,42 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats ) } + db := newDatabase(c.r, c.dbName) + + res, err := db.ListCollections(ctx, new(backends.ListCollectionsParams)) + if err != nil { + return nil, lazyerrors.Error(err) + } + + var cInfo *backends.CollectionInfo + + for _, elem := range res.Collections { + if elem.Name == c.name { + cInfo = &elem + break + } + } + + if cInfo == nil { + return nil, backends.NewError( + backends.ErrorCodeCollectionDoesNotExist, + lazyerrors.Errorf("no ns %s.%s", c.dbName, c.name), + ) + } + stats, err := collectionsStats(ctx, p, c.dbName, []*metadata.Collection{coll}) if err != nil { return nil, lazyerrors.Error(err) } return &backends.CollectionStatsResult{ - CountObjects: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: stats.sizeTables + stats.sizeIndexes, - SizeIndexes: stats.sizeIndexes, - SizeCollection: stats.sizeTables, + CappedSize: cInfo.CappedSize, + CappedDocuments: cInfo.CappedDocuments, + CountObjects: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: stats.sizeTables + stats.sizeIndexes, + SizeIndexes: stats.sizeIndexes, + SizeCollection: stats.sizeTables, }, nil } diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 962a5da60020..605dfb9eb910 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -315,17 +315,42 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats ) } + d := newDatabase(c.r, c.dbName) + + res, err := d.ListCollections(ctx, new(backends.ListCollectionsParams)) + if err != nil { + return nil, lazyerrors.Error(err) + } + + var cInfo *backends.CollectionInfo + + for _, elem := range res.Collections { + if elem.Name == c.name { + cInfo = &elem + break + } + } + + if cInfo == nil { + return nil, backends.NewError( + backends.ErrorCodeCollectionDoesNotExist, + lazyerrors.Errorf("no ns %s.%s", c.dbName, c.name), + ) + } + stats, err := collectionsStats(ctx, db, []*metadata.Collection{coll}) if err != nil { return nil, lazyerrors.Error(err) } return &backends.CollectionStatsResult{ - CountObjects: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: stats.sizeTables + stats.sizeIndexes, - SizeIndexes: stats.sizeIndexes, - SizeCollection: stats.sizeTables, + CappedSize: cInfo.CappedSize, + CappedDocuments: cInfo.CappedDocuments, + CountObjects: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: stats.sizeTables + stats.sizeIndexes, + SizeIndexes: stats.sizeIndexes, + SizeCollection: stats.sizeTables, }, nil } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 59b4ca5b5b84..ce936025739e 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -408,15 +408,15 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "count", collStats.CountObjects, "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, - "freeStorageSize", int64(0), // TODO https://github.com/FerretDB/FerretDB/issues/2342 - "capped", false, // TODO https://github.com/FerretDB/FerretDB/issues/2342 - "wiredTiger", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 + // TODO https://github.com/FerretDB/FerretDB/issues/2447 + "freeStorageSize", int64(0), + "capped", collStats.CappedSize != int64(0), "nindexes", collStats.CountIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "totalIndexSize", collStats.SizeIndexes, "totalSize", collStats.SizeTotal, - "indexSizes", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 + "indexSizes", collStats.SizeIndexes, )), ) } diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 561e5e3e8247..86436b25ca98 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -99,6 +99,9 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs capped := stats.CappedSize != int64(0) + // add freeStorageSize + // TODO https://github.com/FerretDB/FerretDB/issues/2447 + // MongoDB uses "numbers" that could be int32 or int64, // FerretDB always returns int64 for simplicity. pairs = append(pairs, diff --git a/internal/handlers/sqlite/msg_dbstats.go b/internal/handlers/sqlite/msg_dbstats.go index 11b7f4146cc1..6582c9135d6d 100644 --- a/internal/handlers/sqlite/msg_dbstats.go +++ b/internal/handlers/sqlite/msg_dbstats.go @@ -85,6 +85,9 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountObjects) } + // add freeStorageSize, indexFreeStorageSize and totalFreeStorageSize when freeStorage parameter is 1 + // TODO https://github.com/FerretDB/FerretDB/issues/2447 + pairs = append(pairs, "dataSize", stats.SizeCollections/scale, "storageSize", stats.SizeCollections/scale, From 10b8bbdc2ca9730d15baec49e390f00d377c1a47 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 12 Oct 2023 12:41:47 +0900 Subject: [PATCH 13/27] add tests and todos --- integration/aggregate_stats_test.go | 43 +++++++++++++++++++ integration/commands_administration_test.go | 36 +++++++++++++++- .../common/aggregations/stages/collstats.go | 8 +++- internal/handlers/sqlite/msg_aggregate.go | 2 +- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/integration/aggregate_stats_test.go b/integration/aggregate_stats_test.go index 3f53270019e7..7ec52a9e69c2 100644 --- a/integration/aggregate_stats_test.go +++ b/integration/aggregate_stats_test.go @@ -17,13 +17,56 @@ package integration import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "github.com/FerretDB/FerretDB/integration/setup" + "github.com/FerretDB/FerretDB/integration/shareddata" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/must" ) +func TestAggregateCollStats(t *testing.T) { + t.Parallel() + + ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) + + pipeline := bson.A{bson.D{{"$collStats", bson.D{{"storageStats", bson.D{}}}}}} + + cursor, err := collection.Aggregate(ctx, pipeline) + require.NoError(t, err) + + var res []bson.D + err = cursor.All(ctx, &res) + require.NoError(t, err) + + require.Len(t, res, 1) + doc := ConvertDocument(t, res[0]) + + assert.Equal(t, collection.Database().Name()+"."+collection.Name(), must.NotFail(doc.Get("ns"))) + + v, _ := doc.Get("storageStats") + require.NotNil(t, v) + + storageStats, ok := v.(*types.Document) + require.True(t, ok) + + assert.NotZero(t, must.NotFail(storageStats.Get("size"))) + assert.NotZero(t, must.NotFail(storageStats.Get("count"))) + assert.NotZero(t, must.NotFail(storageStats.Get("avgObjSize"))) + assert.NotZero(t, must.NotFail(storageStats.Get("storageSize"))) + // TODO https://github.com/FerretDB/FerretDB/issues/2447 + // assert.NotZero(t, must.NotFail(storageStats.Get("freeStorageSize"))) + assert.Equal(t, false, must.NotFail(storageStats.Get("capped"))) + assert.NotZero(t, must.NotFail(storageStats.Get("nindexes"))) + assert.NotZero(t, must.NotFail(storageStats.Get("totalIndexSize"))) + assert.NotZero(t, must.NotFail(storageStats.Get("totalSize"))) + assert.NotZero(t, must.NotFail(storageStats.Get("indexSizes"))) + assert.Equal(t, int32(1), must.NotFail(storageStats.Get("scaleFactor"))) +} + func TestAggregateCollStatsCommandErrors(t *testing.T) { t.Parallel() diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index c3352fbd4a81..f5da38d1fbd4 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -690,6 +690,8 @@ func TestCommandsAdministrationCollStatsEmpty(t *testing.T) { assert.EqualValues(t, 0, must.NotFail(doc.Get("size"))) assert.EqualValues(t, 0, must.NotFail(doc.Get("count"))) assert.EqualValues(t, 0, must.NotFail(doc.Get("storageSize"))) + // add assertion for freeStorageSize + // TODO https://github.com/FerretDB/FerretDB/issues/2447 assert.EqualValues(t, 0, must.NotFail(doc.Get("nindexes"))) assert.EqualValues(t, 0, must.NotFail(doc.Get("totalIndexSize"))) assert.EqualValues(t, 0, must.NotFail(doc.Get("totalSize"))) @@ -724,6 +726,8 @@ func TestCommandsAdministrationCollStats(t *testing.T) { assert.InDelta(t, 40_000, must.NotFail(doc.Get("size")), 39_900) assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) assert.InDelta(t, 40_000, must.NotFail(doc.Get("storageSize")), 39_900) + // add assertion for freeStorageSize + // TODO https://github.com/FerretDB/FerretDB/issues/2447 assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) assert.InDelta(t, 12_000, must.NotFail(doc.Get("totalIndexSize")), 11_000) assert.InDelta(t, 32_000, must.NotFail(doc.Get("totalSize")), 30_000) @@ -761,6 +765,8 @@ func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { assert.InDelta(t, 16, must.NotFail(doc.Get("size")), 16) assert.InDelta(t, 2_400, must.NotFail(doc.Get("avgObjSize")), 2_370) assert.InDelta(t, 24, must.NotFail(doc.Get("storageSize")), 24) + // add assertion for freeStorageSize + // TODO https://github.com/FerretDB/FerretDB/issues/2447 assert.EqualValues(t, 1, must.NotFail(doc.Get("nindexes"))) assert.InDelta(t, 8, must.NotFail(doc.Get("totalIndexSize")), 8) assert.InDelta(t, 24, must.NotFail(doc.Get("totalSize")), 24) @@ -804,8 +810,6 @@ func TestCommandsAdministrationDataSize(t *testing.T) { assert.Equal(t, float64(1), must.NotFail(doc.Get("ok"))) assert.InDelta(t, 24_576, must.NotFail(doc.Get("size")), 24_576) assert.InDelta(t, 4, must.NotFail(doc.Get("numObjects")), 4) // TODO https://github.com/FerretDB/FerretDB/issues/727 - // reduce delta to 200 once ANALYZE is called less - // TODO https://github.com/FerretDB/FerretDB/issues/3518 assert.InDelta(t, 200, must.NotFail(doc.Get("millis")), 300) }) @@ -904,6 +908,15 @@ func TestCommandsAdministrationDBStats(t *testing.T) { assert.InDelta(t, 37_500, doc.Remove("storageSize"), 37_450) assert.InDelta(t, 49_152, doc.Remove("totalSize"), 49_100) + freeStorageSize, _ := doc.Get("freeStorageSize") + assert.Nil(t, freeStorageSize) + + indexFreeStorageSize, _ := doc.Get("indexFreeStorageSize") + assert.Nil(t, indexFreeStorageSize) + + totalFreeStorageSize, _ := doc.Get("totalFreeStorageSize") + assert.Nil(t, totalFreeStorageSize) + // TODO assert.Empty(t, doc.Keys()) // https://github.com/FerretDB/FerretDB/issues/727 } @@ -980,6 +993,25 @@ func TestCommandsAdministrationDBStatsEmptyWithScale(t *testing.T) { // https://github.com/FerretDB/FerretDB/issues/727 } +func TestCommandsAdministrationDBStatsFreeStorage(t *testing.T) { + t.Parallel() + + ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) + + var actual bson.D + command := bson.D{{"dbStats", int32(1)}, {"freeStorage", int32(1)}} + err := collection.Database().RunCommand(ctx, command).Decode(&actual) + require.NoError(t, err) + + doc := ConvertDocument(t, actual) + + assert.Equal(t, float64(1), doc.Remove("scaleFactor")) + assert.Equal(t, float64(1), doc.Remove("ok")) + + // assert freeStorageSize, indexFreeStorageSize and totalFreeStorageSize + // TODO https://github.com/FerretDB/FerretDB/issues/2447 +} + //nolint:paralleltest // we test a global server status func TestCommandsAdministrationServerStatus(t *testing.T) { ctx, collection := setup.Setup(t) diff --git a/internal/handlers/common/aggregations/stages/collstats.go b/internal/handlers/common/aggregations/stages/collstats.go index eed60663c472..cf5830df61ab 100644 --- a/internal/handlers/common/aggregations/stages/collstats.go +++ b/internal/handlers/common/aggregations/stages/collstats.go @@ -39,7 +39,7 @@ type collStats struct { // storageStats represents $collStats.storageStats field. type storageStats struct { - scale int64 + scale int32 } // newCollStats creates a new $collStats stage. @@ -70,14 +70,18 @@ func newCollStats(stage *types.Document) (aggregations.Stage, error) { storageStatsFields := must.NotFail(fields.Get("storageStats")).(*types.Document) + cs.storageStats.scale = 1 + var s any if s, err = storageStatsFields.Get("scale"); err == nil { - cs.storageStats.scale, err = commonparams.GetValidatedNumberParamWithMinValue( + scale, err := commonparams.GetValidatedNumberParamWithMinValue( "$collStats.storageStats", "scale", s, 1, ) if err != nil { return nil, err } + + cs.storageStats.scale = int32(scale) } } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index ce936025739e..c5dfb0a24cb9 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -416,7 +416,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "totalIndexSize", collStats.SizeIndexes, "totalSize", collStats.SizeTotal, - "indexSizes", collStats.SizeIndexes, + "indexSizes", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 )), ) } From 0bb475c695235ba03baf2635402040d649a04489 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 12 Oct 2023 12:43:38 +0900 Subject: [PATCH 14/27] lint --- integration/commands_administration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index f5da38d1fbd4..44f844f3598b 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -1007,7 +1007,6 @@ func TestCommandsAdministrationDBStatsFreeStorage(t *testing.T) { assert.Equal(t, float64(1), doc.Remove("scaleFactor")) assert.Equal(t, float64(1), doc.Remove("ok")) - // assert freeStorageSize, indexFreeStorageSize and totalFreeStorageSize // TODO https://github.com/FerretDB/FerretDB/issues/2447 } From e88eb6df49c760d57badbf9ef5c1fcc6c26ca888 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 12 Oct 2023 12:58:41 +0900 Subject: [PATCH 15/27] update test --- integration/commands_administration_compat_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index c9d3d8fe9b7b..1c6500e992ed 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -117,9 +117,8 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { }{ "Size": { opts: &options.CreateCollectionOptions{ - Capped: pointer.ToBool(true), - SizeInBytes: pointer.ToInt64(1000), - MaxDocuments: pointer.ToInt64(10), + Capped: pointer.ToBool(true), + SizeInBytes: pointer.ToInt64(1000), }, }, "MaxDocuments": { From d7276def404e3578273748299f03e283692eb123 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Thu, 12 Oct 2023 15:43:11 +0900 Subject: [PATCH 16/27] revert change --- integration/aggregate_stats_test.go | 3 ++- internal/handlers/common/aggregations/stages/collstats.go | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/integration/aggregate_stats_test.go b/integration/aggregate_stats_test.go index 7ec52a9e69c2..737f0d2a9efd 100644 --- a/integration/aggregate_stats_test.go +++ b/integration/aggregate_stats_test.go @@ -64,7 +64,8 @@ func TestAggregateCollStats(t *testing.T) { assert.NotZero(t, must.NotFail(storageStats.Get("totalIndexSize"))) assert.NotZero(t, must.NotFail(storageStats.Get("totalSize"))) assert.NotZero(t, must.NotFail(storageStats.Get("indexSizes"))) - assert.Equal(t, int32(1), must.NotFail(storageStats.Get("scaleFactor"))) + // TODO https://github.com/FerretDB/FerretDB/issues/2447 + // assert.Equal(t, int32(1), must.NotFail(storageStats.Get("scaleFactor"))) } func TestAggregateCollStatsCommandErrors(t *testing.T) { diff --git a/internal/handlers/common/aggregations/stages/collstats.go b/internal/handlers/common/aggregations/stages/collstats.go index cf5830df61ab..eed60663c472 100644 --- a/internal/handlers/common/aggregations/stages/collstats.go +++ b/internal/handlers/common/aggregations/stages/collstats.go @@ -39,7 +39,7 @@ type collStats struct { // storageStats represents $collStats.storageStats field. type storageStats struct { - scale int32 + scale int64 } // newCollStats creates a new $collStats stage. @@ -70,18 +70,14 @@ func newCollStats(stage *types.Document) (aggregations.Stage, error) { storageStatsFields := must.NotFail(fields.Get("storageStats")).(*types.Document) - cs.storageStats.scale = 1 - var s any if s, err = storageStatsFields.Get("scale"); err == nil { - scale, err := commonparams.GetValidatedNumberParamWithMinValue( + cs.storageStats.scale, err = commonparams.GetValidatedNumberParamWithMinValue( "$collStats.storageStats", "scale", s, 1, ) if err != nil { return nil, err } - - cs.storageStats.scale = int32(scale) } } From 90a3622a9642384bf12badc856719d2d6781fced Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 13 Oct 2023 17:56:16 +0900 Subject: [PATCH 17/27] usage of options in test --- .../commands_administration_compat_test.go | 35 ++++++++++++------- internal/handlers/sqlite/msg_aggregate.go | 2 +- internal/handlers/sqlite/msg_collstats.go | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index 1c6500e992ed..27c07ff308fc 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -113,20 +113,18 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { ctx, targetCollection, compatCollection := s.Ctx, s.TargetCollections[0], s.CompatCollections[0] for name, tc := range map[string]struct { //nolint:vet // for readability - opts *options.CreateCollectionOptions + capped *bool + sizeInBytes *int64 + maxDocuments *int64 }{ "Size": { - opts: &options.CreateCollectionOptions{ - Capped: pointer.ToBool(true), - SizeInBytes: pointer.ToInt64(1000), - }, + capped: pointer.ToBool(true), + sizeInBytes: pointer.ToInt64(1000), }, "MaxDocuments": { - opts: &options.CreateCollectionOptions{ - Capped: pointer.ToBool(true), - SizeInBytes: pointer.ToInt64(1000), - MaxDocuments: pointer.ToInt64(10), - }, + capped: pointer.ToBool(true), + sizeInBytes: pointer.ToInt64(1000), + maxDocuments: pointer.ToInt64(10), }, } { name, tc := name, tc @@ -134,10 +132,23 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { t.Parallel() cName := testutil.CollectionName(t) + name - targetErr := targetCollection.Database().CreateCollection(ctx, cName, tc.opts) + opts := options.CreateCollection() + if tc.capped != nil { + opts.SetCapped(*tc.capped) + } + + if tc.sizeInBytes != nil { + opts.SetSizeInBytes(*tc.sizeInBytes) + } + + if tc.maxDocuments != nil { + opts.SetMaxDocuments(*tc.maxDocuments) + } + + targetErr := targetCollection.Database().CreateCollection(ctx, cName, opts) require.NoError(t, targetErr) - compatErr := compatCollection.Database().CreateCollection(ctx, cName, tc.opts) + compatErr := compatCollection.Database().CreateCollection(ctx, cName, opts) require.NoError(t, compatErr) require.Equal(t, compatCollection.Name(), targetCollection.Name()) diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index c5dfb0a24cb9..0403aea9c530 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -410,7 +410,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "storageSize", collStats.SizeCollection, // TODO https://github.com/FerretDB/FerretDB/issues/2447 "freeStorageSize", int64(0), - "capped", collStats.CappedSize != int64(0), + "capped", collStats.CappedSize > 0, "nindexes", collStats.CountIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 86436b25ca98..fc9a4b9b9e03 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -97,7 +97,7 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) } - capped := stats.CappedSize != int64(0) + capped := stats.CappedSize > 0 // add freeStorageSize // TODO https://github.com/FerretDB/FerretDB/issues/2447 From 4a5e4ebd9fa74c0cd1cf168513aeca2ea27055b2 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 13 Oct 2023 18:19:46 +0900 Subject: [PATCH 18/27] rename --- internal/backends/collection.go | 4 ++-- internal/backends/collection_test.go | 2 +- internal/backends/postgresql/collection.go | 2 +- internal/backends/sqlite/collection.go | 2 +- internal/handlers/pg/msg_aggregate.go | 8 ++++---- internal/handlers/pg/msg_collstats.go | 6 +++--- internal/handlers/pg/msg_datasize.go | 2 +- internal/handlers/pg/pgdb/stats.go | 4 ++-- internal/handlers/sqlite/msg_aggregate.go | 8 ++++---- internal/handlers/sqlite/msg_collstats.go | 6 +++--- internal/handlers/sqlite/msg_datasize.go | 2 +- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 2781ea0645de..156b41f1ff38 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -228,11 +228,11 @@ type CollectionStatsParams struct { // CollectionStatsResult represents the results of Collection.Stats method. // // CappedSize and CappedDocuments are zero for not capped collection. -// CountObjects is an estimate of the number of documents. +// CountDocuments is an estimate of the number of documents. type CollectionStatsResult struct { CappedSize int64 CappedDocuments int64 - CountObjects int64 + CountDocuments int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index 1b0a9328f124..d417279f5e62 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -213,7 +213,7 @@ func TestCollectionStats(t *testing.T) { require.Less(t, res.SizeTotal, dbStatsRes.SizeTotal) require.NotZero(t, res.SizeCollection) require.Less(t, res.SizeCollection, dbStatsRes.SizeCollections) - require.Equal(t, res.CountObjects, int64(1)) + require.Equal(t, res.CountDocuments, int64(1)) require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) }) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index dd4bd2750345..4521bcffe876 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -372,7 +372,7 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats return &backends.CollectionStatsResult{ CappedSize: cInfo.CappedSize, CappedDocuments: cInfo.CappedDocuments, - CountObjects: stats.countRows, + CountDocuments: stats.countRows, CountIndexes: stats.countIndexes, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index ff5bbec304f3..b489bc34074a 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -346,7 +346,7 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats return &backends.CollectionStatsResult{ CappedSize: cInfo.CappedSize, CappedDocuments: cInfo.CappedDocuments, - CountObjects: stats.countRows, + CountDocuments: stats.countRows, CountIndexes: stats.countIndexes, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, diff --git a/internal/handlers/pg/msg_aggregate.go b/internal/handlers/pg/msg_aggregate.go index 4f492a6e70a3..e7014745fd99 100644 --- a/internal/handlers/pg/msg_aggregate.go +++ b/internal/handlers/pg/msg_aggregate.go @@ -428,14 +428,14 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasStorage { var avgObjSize int64 - if collStats.CountObjects > 0 { - avgObjSize = collStats.SizeCollection / collStats.CountObjects + if collStats.CountDocuments > 0 { + avgObjSize = collStats.SizeCollection / collStats.CountDocuments } doc.Set( "storageStats", must.NotFail(types.NewDocument( "size", collStats.SizeTotal, - "count", collStats.CountObjects, + "count", collStats.CountDocuments, "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, "freeStorageSize", int64(0), // TODO https://github.com/FerretDB/FerretDB/issues/2342 @@ -453,7 +453,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasCount { doc.Set( - "count", collStats.CountObjects, + "count", collStats.CountDocuments, ) } diff --git a/internal/handlers/pg/msg_collstats.go b/internal/handlers/pg/msg_collstats.go index 96d54ad4210c..0fc59ba0e80d 100644 --- a/internal/handlers/pg/msg_collstats.go +++ b/internal/handlers/pg/msg_collstats.go @@ -84,12 +84,12 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs := []any{ "ns", db + "." + collection, "size", stats.SizeCollection / int64(scale), - "count", stats.CountObjects, + "count", stats.CountDocuments, } // If there are objects in the collection, calculate the average object size. - if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) + if stats.CountDocuments > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountDocuments) } // MongoDB uses "numbers" that could be int32 or int64, diff --git a/internal/handlers/pg/msg_datasize.go b/internal/handlers/pg/msg_datasize.go index 9e8e485faae4..7bd6327bb13e 100644 --- a/internal/handlers/pg/msg_datasize.go +++ b/internal/handlers/pg/msg_datasize.go @@ -90,7 +90,7 @@ func (h *Handler) MsgDataSize(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg } pairs = append(pairs, "size", stats.SizeTotal, - "numObjects", stats.CountObjects, + "numObjects", stats.CountDocuments, "millis", int32(elapses.Milliseconds()), "ok", float64(1), ) diff --git a/internal/handlers/pg/pgdb/stats.go b/internal/handlers/pg/pgdb/stats.go index 3a9039298767..e64befc21679 100644 --- a/internal/handlers/pg/pgdb/stats.go +++ b/internal/handlers/pg/pgdb/stats.go @@ -53,7 +53,7 @@ type DBStats struct { // Include more data. // TODO https://github.com/FerretDB/FerretDB/issues/2447 type CollStats struct { - CountObjects int64 + CountDocuments int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 @@ -175,7 +175,7 @@ func CalculateCollStats(ctx context.Context, tx pgx.Tx, db, collection string) ( ) row := tx.QueryRow(ctx, sql) - if err := row.Scan(&res.CountObjects, &res.SizeTotal, &res.SizeCollection, &res.SizeIndexes); err != nil { + if err := row.Scan(&res.CountDocuments, &res.SizeTotal, &res.SizeCollection, &res.SizeIndexes); err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index 0403aea9c530..b7e44d84e08e 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -398,14 +398,14 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasStorage { var avgObjSize int64 - if collStats.CountObjects > 0 { - avgObjSize = collStats.SizeCollection / collStats.CountObjects + if collStats.CountDocuments > 0 { + avgObjSize = collStats.SizeCollection / collStats.CountDocuments } doc.Set( "storageStats", must.NotFail(types.NewDocument( "size", collStats.SizeTotal, - "count", collStats.CountObjects, + "count", collStats.CountDocuments, "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, // TODO https://github.com/FerretDB/FerretDB/issues/2447 @@ -423,7 +423,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasCount { doc.Set( - "count", collStats.CountObjects, + "count", collStats.CountDocuments, ) } diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index fc9a4b9b9e03..60aa59eb2df2 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -89,12 +89,12 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs := []any{ "ns", dbName + "." + collection, "size", stats.SizeCollection / scale, - "count", stats.CountObjects, + "count", stats.CountDocuments, } // If there are objects in the collection, calculate the average object size. - if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) + if stats.CountDocuments > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountDocuments) } capped := stats.CappedSize > 0 diff --git a/internal/handlers/sqlite/msg_datasize.go b/internal/handlers/sqlite/msg_datasize.go index a0b15a7feafa..f4b22ec42634 100644 --- a/internal/handlers/sqlite/msg_datasize.go +++ b/internal/handlers/sqlite/msg_datasize.go @@ -99,7 +99,7 @@ func (h *Handler) MsgDataSize(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg Documents: []*types.Document{must.NotFail(types.NewDocument( "estimate", false, "size", stats.SizeTotal, - "numObjects", stats.CountObjects, + "numObjects", stats.CountDocuments, "millis", int32(time.Since(started).Milliseconds()), "ok", float64(1), ))}, From 4847e83ccdf90906c36416c170105b51dc1e509f Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 13 Oct 2023 18:52:44 +0900 Subject: [PATCH 19/27] capped information is fetched from list collection --- internal/backends/collection.go | 13 ++++---- internal/backends/postgresql/collection.go | 35 ++++------------------ internal/backends/sqlite/collection.go | 35 ++++------------------ internal/handlers/sqlite/msg_aggregate.go | 25 ++++++++++++++-- internal/handlers/sqlite/msg_collstats.go | 29 ++++++++++++++---- 5 files changed, 61 insertions(+), 76 deletions(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 156b41f1ff38..9628ffb8fff7 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -227,16 +227,13 @@ type CollectionStatsParams struct { // CollectionStatsResult represents the results of Collection.Stats method. // -// CappedSize and CappedDocuments are zero for not capped collection. // CountDocuments is an estimate of the number of documents. type CollectionStatsResult struct { - CappedSize int64 - CappedDocuments int64 - CountDocuments int64 - CountIndexes int64 - SizeTotal int64 - SizeIndexes int64 - SizeCollection int64 + CountDocuments int64 + CountIndexes int64 + SizeTotal int64 + SizeIndexes int64 + SizeCollection int64 } // Stats returns statistics about the collection. diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 4521bcffe876..8bb4f10b1b48 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -341,42 +341,17 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats ) } - db := newDatabase(c.r, c.dbName) - - res, err := db.ListCollections(ctx, new(backends.ListCollectionsParams)) - if err != nil { - return nil, lazyerrors.Error(err) - } - - var cInfo *backends.CollectionInfo - - for _, elem := range res.Collections { - if elem.Name == c.name { - cInfo = &elem - break - } - } - - if cInfo == nil { - return nil, backends.NewError( - backends.ErrorCodeCollectionDoesNotExist, - lazyerrors.Errorf("no ns %s.%s", c.dbName, c.name), - ) - } - stats, err := collectionsStats(ctx, p, c.dbName, []*metadata.Collection{coll}) if err != nil { return nil, lazyerrors.Error(err) } return &backends.CollectionStatsResult{ - CappedSize: cInfo.CappedSize, - CappedDocuments: cInfo.CappedDocuments, - CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: stats.sizeTables + stats.sizeIndexes, - SizeIndexes: stats.sizeIndexes, - SizeCollection: stats.sizeTables, + CountDocuments: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: stats.sizeTables + stats.sizeIndexes, + SizeIndexes: stats.sizeIndexes, + SizeCollection: stats.sizeTables, }, nil } diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index b489bc34074a..7b403efd87f3 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -315,42 +315,17 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats ) } - d := newDatabase(c.r, c.dbName) - - res, err := d.ListCollections(ctx, new(backends.ListCollectionsParams)) - if err != nil { - return nil, lazyerrors.Error(err) - } - - var cInfo *backends.CollectionInfo - - for _, elem := range res.Collections { - if elem.Name == c.name { - cInfo = &elem - break - } - } - - if cInfo == nil { - return nil, backends.NewError( - backends.ErrorCodeCollectionDoesNotExist, - lazyerrors.Errorf("no ns %s.%s", c.dbName, c.name), - ) - } - stats, err := collectionsStats(ctx, db, []*metadata.Collection{coll}) if err != nil { return nil, lazyerrors.Error(err) } return &backends.CollectionStatsResult{ - CappedSize: cInfo.CappedSize, - CappedDocuments: cInfo.CappedDocuments, - CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: stats.sizeTables + stats.sizeIndexes, - SizeIndexes: stats.sizeIndexes, - SizeCollection: stats.sizeTables, + CountDocuments: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: stats.sizeTables + stats.sizeIndexes, + SizeIndexes: stats.sizeIndexes, + SizeCollection: stats.sizeTables, }, nil } diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index b7e44d84e08e..ab235511b41d 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -267,7 +267,7 @@ func (h *Handler) MsgAggregate(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs statistics := stages.GetStatistics(collStatsDocuments) iter, err = processStagesStats(ctx, closer, &stagesStatsParams{ - c, dbName, cName, statistics, collStatsDocuments, + c, db, dbName, cName, statistics, collStatsDocuments, }) } @@ -350,6 +350,7 @@ func processStagesDocuments(ctx context.Context, closer *iterator.MultiCloser, p // stagesStatsParams contains the parameters for processStagesStats. type stagesStatsParams struct { c backends.Collection + db backends.Database dbName string cName string statistics map[stages.Statistic]struct{} @@ -380,6 +381,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st )) var collStats *backends.CollectionStatsResult + var cInfo backends.CollectionInfo if hasCount || hasStorage { collStats, err = p.c.Stats(ctx, new(backends.CollectionStatsParams)) @@ -394,6 +396,25 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if err != nil { return nil, lazyerrors.Error(err) } + + var list *backends.ListCollectionsResult + + if list, err = p.db.ListCollections(ctx, new(backends.ListCollectionsParams)); err != nil { + return nil, lazyerrors.Error(err) + } + + var found bool + + for _, cInfo := range list.Collections { + if cInfo.Name == p.cName { + found = true + break + } + } + + if !found { + cInfo = backends.CollectionInfo{} + } } if hasStorage { @@ -410,7 +431,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st "storageSize", collStats.SizeCollection, // TODO https://github.com/FerretDB/FerretDB/issues/2447 "freeStorageSize", int64(0), - "capped", collStats.CappedSize > 0, + "capped", cInfo.Capped(), "nindexes", collStats.CountIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 60aa59eb2df2..906bcac58772 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -76,6 +76,25 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs return nil, lazyerrors.Error(err) } + list, err := db.ListCollections(ctx, new(backends.ListCollectionsParams)) + if err != nil { + return nil, lazyerrors.Error(err) + } + + var found bool + var cInfo backends.CollectionInfo + + for _, cInfo := range list.Collections { + if cInfo.Name == collection { + found = true + break + } + } + + if !found { + cInfo = backends.CollectionInfo{} + } + stats, err := c.Stats(ctx, new(backends.CollectionStatsParams)) if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { stats = new(backends.CollectionStatsResult) @@ -97,8 +116,6 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountDocuments) } - capped := stats.CappedSize > 0 - // add freeStorageSize // TODO https://github.com/FerretDB/FerretDB/issues/2447 @@ -110,13 +127,13 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs "totalIndexSize", stats.SizeIndexes/scale, "totalSize", stats.SizeTotal/scale, "scaleFactor", int32(scale), - "capped", capped, + "capped", cInfo.Capped(), ) - if capped { + if cInfo.Capped() { pairs = append(pairs, - "max", stats.CappedDocuments, - "maxSize", stats.CappedSize/scale, + "max", cInfo.CappedDocuments, + "maxSize", cInfo.CappedSize/scale, ) } From 9032ee44b33a8ff1821ed7fb4da1f2c85bfd1405 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 13 Oct 2023 18:57:15 +0900 Subject: [PATCH 20/27] dbstats uses countdocument too --- internal/backends/database.go | 4 ++-- internal/backends/postgresql/database.go | 2 +- internal/backends/postgresql/database_test.go | 4 ++-- internal/backends/sqlite/database.go | 2 +- internal/backends/sqlite/database_test.go | 2 +- internal/handlers/pg/msg_dbstats.go | 6 +++--- internal/handlers/pg/pgdb/stats.go | 4 ++-- internal/handlers/sqlite/msg_dbstats.go | 6 +++--- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/backends/database.go b/internal/backends/database.go index 9ecdce0fa152..b182b6401df4 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -193,12 +193,12 @@ type DatabaseStatsParams struct { // DatabaseStatsResult represents the results of Database.Stats method. // -// CountObjects is an estimate of the number of documents. +// CountDocuments is an estimate of the number of documents. // // TODO https://github.com/FerretDB/FerretDB/issues/2447 type DatabaseStatsResult struct { CountCollections int64 - CountObjects int64 + CountDocuments int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 diff --git a/internal/backends/postgresql/database.go b/internal/backends/postgresql/database.go index 2f46330fef61..79b9e7b8d08d 100644 --- a/internal/backends/postgresql/database.go +++ b/internal/backends/postgresql/database.go @@ -173,7 +173,7 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar return &backends.DatabaseStatsResult{ CountCollections: int64(len(list)), - CountObjects: stats.countRows, + CountDocuments: stats.countRows, CountIndexes: stats.countIndexes, SizeTotal: *schemaSize, SizeIndexes: stats.sizeIndexes, diff --git a/internal/backends/postgresql/database_test.go b/internal/backends/postgresql/database_test.go index 5b3d8f9c173f..89c3c97561d0 100644 --- a/internal/backends/postgresql/database_test.go +++ b/internal/backends/postgresql/database_test.go @@ -70,7 +70,7 @@ func TestDatabaseStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Equal(t, res.CountCollections, int64(len(cNames))) require.Zero(t, res.SizeCollections) - require.Zero(t, res.CountObjects) + require.Zero(t, res.CountDocuments) require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) // includes metadata table's indexes }) @@ -89,7 +89,7 @@ func TestDatabaseStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Equal(t, res.CountCollections, int64(len(cNames))) require.NotZero(t, res.SizeCollections) - require.Equal(t, int64(1), res.CountObjects) + require.Equal(t, int64(1), res.CountDocuments) require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) }) diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 37fe462151df..a384942fcf32 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -149,7 +149,7 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar return &backends.DatabaseStatsResult{ CountCollections: int64(len(list)), - CountObjects: stats.countRows, + CountDocuments: stats.countRows, CountIndexes: stats.countIndexes, SizeTotal: totalSize, SizeIndexes: stats.sizeIndexes, diff --git a/internal/backends/sqlite/database_test.go b/internal/backends/sqlite/database_test.go index 3d5ebcdcf99e..67a582e336bf 100644 --- a/internal/backends/sqlite/database_test.go +++ b/internal/backends/sqlite/database_test.go @@ -51,6 +51,6 @@ func TestDatabaseStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Equal(t, res.CountCollections, int64(len(cNames))) require.NotZero(t, res.SizeCollections) - require.Zero(t, res.CountObjects) + require.Zero(t, res.CountDocuments) }) } diff --git a/internal/handlers/pg/msg_dbstats.go b/internal/handlers/pg/msg_dbstats.go index 8a484dec9145..c671fc4b1aa4 100644 --- a/internal/handlers/pg/msg_dbstats.go +++ b/internal/handlers/pg/msg_dbstats.go @@ -70,11 +70,11 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, "collections", stats.CountCollections, // TODO https://github.com/FerretDB/FerretDB/issues/176 "views", int32(0), - "objects", stats.CountObjects, + "objects", stats.CountDocuments, } - if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountObjects) + if stats.CountDocuments > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountDocuments) } // MongoDB uses "numbers" that could be int32 or int64, diff --git a/internal/handlers/pg/pgdb/stats.go b/internal/handlers/pg/pgdb/stats.go index e64befc21679..8fac3dddcfd9 100644 --- a/internal/handlers/pg/pgdb/stats.go +++ b/internal/handlers/pg/pgdb/stats.go @@ -38,7 +38,7 @@ type ServerStats struct { // TODO https://github.com/FerretDB/FerretDB/issues/2447 type DBStats struct { CountCollections int64 - CountObjects int64 + CountDocuments int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 @@ -131,7 +131,7 @@ func CalculateDBStats(ctx context.Context, tx pgx.Tx, db string) (*DBStats, erro row = tx.QueryRow(ctx, sql, args...) if err := row.Scan( - &res.CountCollections, &res.CountIndexes, &res.CountObjects, &res.SizeCollections, &res.SizeIndexes, + &res.CountCollections, &res.CountIndexes, &res.CountDocuments, &res.SizeCollections, &res.SizeIndexes, ); err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/handlers/sqlite/msg_dbstats.go b/internal/handlers/sqlite/msg_dbstats.go index 6582c9135d6d..a82e72cc6093 100644 --- a/internal/handlers/sqlite/msg_dbstats.go +++ b/internal/handlers/sqlite/msg_dbstats.go @@ -78,11 +78,11 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, "collections", stats.CountCollections, // TODO https://github.com/FerretDB/FerretDB/issues/176 "views", int32(0), - "objects", stats.CountObjects, + "objects", stats.CountDocuments, } - if stats.CountObjects > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountObjects) + if stats.CountDocuments > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountDocuments) } // add freeStorageSize, indexFreeStorageSize and totalFreeStorageSize when freeStorage parameter is 1 From 7695485b4a4ee885b760cfe9d05b9b5076c856b8 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 13 Oct 2023 19:30:12 +0900 Subject: [PATCH 21/27] remove collection count from stats --- internal/backends/database.go | 11 +++++------ internal/backends/postgresql/database.go | 11 +++++------ internal/backends/postgresql/database_test.go | 2 -- internal/backends/sqlite/database.go | 11 +++++------ internal/backends/sqlite/database_test.go | 1 - internal/handlers/sqlite/msg_dbstats.go | 7 ++++++- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/internal/backends/database.go b/internal/backends/database.go index b182b6401df4..d3c338fe425f 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -197,12 +197,11 @@ type DatabaseStatsParams struct { // // TODO https://github.com/FerretDB/FerretDB/issues/2447 type DatabaseStatsResult struct { - CountCollections int64 - CountDocuments int64 - CountIndexes int64 - SizeTotal int64 - SizeIndexes int64 - SizeCollections int64 + CountDocuments int64 + CountIndexes int64 + SizeTotal int64 + SizeIndexes int64 + SizeCollections int64 } // Stats returns statistics about the database. diff --git a/internal/backends/postgresql/database.go b/internal/backends/postgresql/database.go index 79b9e7b8d08d..d69b8db6a767 100644 --- a/internal/backends/postgresql/database.go +++ b/internal/backends/postgresql/database.go @@ -172,12 +172,11 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar } return &backends.DatabaseStatsResult{ - CountCollections: int64(len(list)), - CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: *schemaSize, - SizeIndexes: stats.sizeIndexes, - SizeCollections: stats.sizeTables, + CountDocuments: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: *schemaSize, + SizeIndexes: stats.sizeIndexes, + SizeCollections: stats.sizeTables, }, nil } diff --git a/internal/backends/postgresql/database_test.go b/internal/backends/postgresql/database_test.go index 89c3c97561d0..c7451e78b635 100644 --- a/internal/backends/postgresql/database_test.go +++ b/internal/backends/postgresql/database_test.go @@ -68,7 +68,6 @@ func TestDatabaseStats(t *testing.T) { res, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) require.NoError(t, err) require.NotZero(t, res.SizeTotal) - require.Equal(t, res.CountCollections, int64(len(cNames))) require.Zero(t, res.SizeCollections) require.Zero(t, res.CountDocuments) require.NotZero(t, res.CountIndexes) @@ -87,7 +86,6 @@ func TestDatabaseStats(t *testing.T) { res, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) require.NoError(t, err) require.NotZero(t, res.SizeTotal) - require.Equal(t, res.CountCollections, int64(len(cNames))) require.NotZero(t, res.SizeCollections) require.Equal(t, int64(1), res.CountDocuments) require.NotZero(t, res.CountIndexes) diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index a384942fcf32..208d11090eba 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -148,12 +148,11 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar } return &backends.DatabaseStatsResult{ - CountCollections: int64(len(list)), - CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, - SizeTotal: totalSize, - SizeIndexes: stats.sizeIndexes, - SizeCollections: stats.sizeTables, + CountDocuments: stats.countRows, + CountIndexes: stats.countIndexes, + SizeTotal: totalSize, + SizeIndexes: stats.sizeIndexes, + SizeCollections: stats.sizeTables, }, nil } diff --git a/internal/backends/sqlite/database_test.go b/internal/backends/sqlite/database_test.go index 67a582e336bf..928e77ce702f 100644 --- a/internal/backends/sqlite/database_test.go +++ b/internal/backends/sqlite/database_test.go @@ -49,7 +49,6 @@ func TestDatabaseStats(t *testing.T) { res, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) require.NoError(t, err) require.NotZero(t, res.SizeTotal) - require.Equal(t, res.CountCollections, int64(len(cNames))) require.NotZero(t, res.SizeCollections) require.Zero(t, res.CountDocuments) }) diff --git a/internal/handlers/sqlite/msg_dbstats.go b/internal/handlers/sqlite/msg_dbstats.go index a82e72cc6093..8743fd097d72 100644 --- a/internal/handlers/sqlite/msg_dbstats.go +++ b/internal/handlers/sqlite/msg_dbstats.go @@ -61,6 +61,11 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, lazyerrors.Error(err) } + list, err := db.ListCollections(ctx, new(backends.ListCollectionsParams)) + if err != nil { + return nil, lazyerrors.Error(err) + } + stats, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseDoesNotExist) { stats = new(backends.DatabaseStatsResult) @@ -75,7 +80,7 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, // FerretDB always returns int64 for simplicity. pairs := []any{ "db", dbName, - "collections", stats.CountCollections, + "collections", int64(len(list.Collections)), // TODO https://github.com/FerretDB/FerretDB/issues/176 "views", int32(0), "objects", stats.CountDocuments, From 160854a5f06d28c731aafc638f07a5d8ddde9137 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 11:32:40 +0900 Subject: [PATCH 22/27] cleanup --- .../commands_administration_compat_test.go | 28 ++++++++----------- internal/backends/collection.go | 2 -- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index 27c07ff308fc..10b1db51b785 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -18,7 +18,6 @@ import ( "math" "testing" - "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" @@ -113,18 +112,15 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { ctx, targetCollection, compatCollection := s.Ctx, s.TargetCollections[0], s.CompatCollections[0] for name, tc := range map[string]struct { //nolint:vet // for readability - capped *bool - sizeInBytes *int64 - maxDocuments *int64 + sizeInBytes int64 + maxDocuments int64 }{ "Size": { - capped: pointer.ToBool(true), - sizeInBytes: pointer.ToInt64(1000), + sizeInBytes: 1000, }, "MaxDocuments": { - capped: pointer.ToBool(true), - sizeInBytes: pointer.ToInt64(1000), - maxDocuments: pointer.ToInt64(10), + sizeInBytes: 1000, + maxDocuments: 10, }, } { name, tc := name, tc @@ -133,16 +129,14 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { cName := testutil.CollectionName(t) + name opts := options.CreateCollection() - if tc.capped != nil { - opts.SetCapped(*tc.capped) - } - if tc.sizeInBytes != nil { - opts.SetSizeInBytes(*tc.sizeInBytes) - } + if tc.sizeInBytes > 0 { + opts.SetCapped(true) + opts.SetSizeInBytes(tc.sizeInBytes) - if tc.maxDocuments != nil { - opts.SetMaxDocuments(*tc.maxDocuments) + if tc.maxDocuments > 0 { + opts.SetMaxDocuments(tc.maxDocuments) + } } targetErr := targetCollection.Database().CreateCollection(ctx, cName, opts) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index 09de6229ff31..d56d28c66b8e 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -227,8 +227,6 @@ type CollectionStatsParams struct { } // CollectionStatsResult represents the results of Collection.Stats method. -// -// CountDocuments is an estimate of the number of documents. type CollectionStatsResult struct { CountDocuments int64 CountIndexes int64 From 4c92c093e5a6bd9d2f210b7b6980e00a563b91ce Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 12:09:06 +0900 Subject: [PATCH 23/27] remove CountIndexes from CollStats and DBStats --- internal/backends/collection.go | 1 - internal/backends/collection_test.go | 1 - internal/backends/database.go | 3 --- internal/backends/postgresql/collection.go | 1 - internal/backends/postgresql/database.go | 1 - internal/backends/postgresql/database_test.go | 2 -- internal/backends/postgresql/postgresql.go | 8 +++--- internal/backends/sqlite/collection.go | 1 - internal/backends/sqlite/database.go | 1 - internal/backends/sqlite/sqlite.go | 9 +++---- internal/handlers/sqlite/msg_aggregate.go | 23 +++++++++++++--- internal/handlers/sqlite/msg_collstats.go | 12 ++++++++- internal/handlers/sqlite/msg_dbstats.go | 27 ++++++++++++++++++- 13 files changed, 62 insertions(+), 28 deletions(-) diff --git a/internal/backends/collection.go b/internal/backends/collection.go index d56d28c66b8e..316496a64a8d 100644 --- a/internal/backends/collection.go +++ b/internal/backends/collection.go @@ -229,7 +229,6 @@ type CollectionStatsParams struct { // CollectionStatsResult represents the results of Collection.Stats method. type CollectionStatsResult struct { CountDocuments int64 - CountIndexes int64 SizeTotal int64 SizeIndexes int64 SizeCollection int64 diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index 822eb391e117..41eeffd257b6 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -214,7 +214,6 @@ func TestCollectionStats(t *testing.T) { require.NotZero(t, res.SizeCollection) require.Less(t, res.SizeCollection, dbStatsRes.SizeCollections) require.Equal(t, res.CountDocuments, int64(1)) - require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) }) }) diff --git a/internal/backends/database.go b/internal/backends/database.go index d3c338fe425f..ddf2cdb7aa79 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -193,12 +193,9 @@ type DatabaseStatsParams struct { // DatabaseStatsResult represents the results of Database.Stats method. // -// CountDocuments is an estimate of the number of documents. -// // TODO https://github.com/FerretDB/FerretDB/issues/2447 type DatabaseStatsResult struct { CountDocuments int64 - CountIndexes int64 SizeTotal int64 SizeIndexes int64 SizeCollections int64 diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index 8bb4f10b1b48..ec0096b2b033 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -348,7 +348,6 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats return &backends.CollectionStatsResult{ CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, SizeCollection: stats.sizeTables, diff --git a/internal/backends/postgresql/database.go b/internal/backends/postgresql/database.go index d69b8db6a767..d9d7becdfbba 100644 --- a/internal/backends/postgresql/database.go +++ b/internal/backends/postgresql/database.go @@ -173,7 +173,6 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar return &backends.DatabaseStatsResult{ CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, SizeTotal: *schemaSize, SizeIndexes: stats.sizeIndexes, SizeCollections: stats.sizeTables, diff --git a/internal/backends/postgresql/database_test.go b/internal/backends/postgresql/database_test.go index c7451e78b635..f16ed4e11fa8 100644 --- a/internal/backends/postgresql/database_test.go +++ b/internal/backends/postgresql/database_test.go @@ -70,7 +70,6 @@ func TestDatabaseStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Zero(t, res.SizeCollections) require.Zero(t, res.CountDocuments) - require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) // includes metadata table's indexes }) @@ -88,7 +87,6 @@ func TestDatabaseStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.NotZero(t, res.SizeCollections) require.Equal(t, int64(1), res.CountDocuments) - require.NotZero(t, res.CountIndexes) require.NotZero(t, res.SizeIndexes) }) } diff --git a/internal/backends/postgresql/postgresql.go b/internal/backends/postgresql/postgresql.go index af4c465f04e2..cd4f60c472f4 100644 --- a/internal/backends/postgresql/postgresql.go +++ b/internal/backends/postgresql/postgresql.go @@ -32,10 +32,9 @@ import ( // stats represents information about statistics of tables and indexes. type stats struct { - countRows int64 - countIndexes int64 - sizeIndexes int64 - sizeTables int64 + countRows int64 + sizeIndexes int64 + sizeTables int64 } // collectionsStats returns statistics about tables and indexes for the given collections. @@ -62,7 +61,6 @@ func collectionsStats(ctx context.Context, p *pgxpool.Pool, dbName string, list placeholder.Next() for i, c := range list { - s.countIndexes += int64(len(c.Indexes)) placeholders[i] = placeholder.Next() args = append(args, c.TableName) } diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 7b403efd87f3..5a2cb7f8708e 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -322,7 +322,6 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats return &backends.CollectionStatsResult{ CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, SizeCollection: stats.sizeTables, diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index 208d11090eba..b44da4a5fc07 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -149,7 +149,6 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar return &backends.DatabaseStatsResult{ CountDocuments: stats.countRows, - CountIndexes: stats.countIndexes, SizeTotal: totalSize, SizeIndexes: stats.sizeIndexes, SizeCollections: stats.sizeTables, diff --git a/internal/backends/sqlite/sqlite.go b/internal/backends/sqlite/sqlite.go index 473f19a19440..80cb37fb5ba6 100644 --- a/internal/backends/sqlite/sqlite.go +++ b/internal/backends/sqlite/sqlite.go @@ -42,10 +42,9 @@ import ( // stats represents information about statistics of tables and indexes. type stats struct { - countRows int64 - countIndexes int64 - sizeIndexes int64 - sizeTables int64 + countRows int64 + sizeIndexes int64 + sizeTables int64 } // collectionsStats returns statistics about tables and indexes for the given collections. @@ -106,8 +105,6 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect return nil, lazyerrors.Error(err) } - stats.countIndexes = indexes - placeholders = make([]string, 0, indexes) args = make([]any, 0, indexes) diff --git a/internal/handlers/sqlite/msg_aggregate.go b/internal/handlers/sqlite/msg_aggregate.go index a97a74f29f39..6162d80e3519 100644 --- a/internal/handlers/sqlite/msg_aggregate.go +++ b/internal/handlers/sqlite/msg_aggregate.go @@ -380,6 +380,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st var collStats *backends.CollectionStatsResult var cInfo backends.CollectionInfo + var nIndexes int64 if hasCount || hasStorage { collStats, err = p.c.Stats(ctx, new(backends.CollectionStatsParams)) @@ -395,15 +396,15 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st return nil, lazyerrors.Error(err) } - var list *backends.ListCollectionsResult + var cList *backends.ListCollectionsResult - if list, err = p.db.ListCollections(ctx, new(backends.ListCollectionsParams)); err != nil { + if cList, err = p.db.ListCollections(ctx, new(backends.ListCollectionsParams)); err != nil { return nil, lazyerrors.Error(err) } var found bool - for _, cInfo := range list.Collections { + for _, cInfo := range cList.Collections { if cInfo.Name == p.cName { found = true break @@ -413,6 +414,20 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if !found { cInfo = backends.CollectionInfo{} } + + var iList *backends.ListIndexesResult + + iList, err = p.c.ListIndexes(ctx, new(backends.ListIndexesParams)) + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { + iList = new(backends.ListIndexesResult) + err = nil + } + + if err != nil { + return nil, lazyerrors.Error(err) + } + + nIndexes = int64(len(iList.Indexes)) } if hasStorage { @@ -430,7 +445,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st // TODO https://github.com/FerretDB/FerretDB/issues/2447 "freeStorageSize", int64(0), "capped", cInfo.Capped(), - "nindexes", collStats.CountIndexes, + "nindexes", nIndexes, "indexDetails", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "indexBuilds", must.NotFail(types.NewDocument()), // TODO https://github.com/FerretDB/FerretDB/issues/2342 "totalIndexSize", collStats.SizeIndexes, diff --git a/internal/handlers/sqlite/msg_collstats.go b/internal/handlers/sqlite/msg_collstats.go index 906bcac58772..459a5fb01653 100644 --- a/internal/handlers/sqlite/msg_collstats.go +++ b/internal/handlers/sqlite/msg_collstats.go @@ -95,6 +95,16 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs cInfo = backends.CollectionInfo{} } + indexes, err := c.ListIndexes(ctx, new(backends.ListIndexesParams)) + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { + indexes = new(backends.ListIndexesResult) + err = nil + } + + if err != nil { + return nil, lazyerrors.Error(err) + } + stats, err := c.Stats(ctx, new(backends.CollectionStatsParams)) if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { stats = new(backends.CollectionStatsResult) @@ -123,7 +133,7 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs // FerretDB always returns int64 for simplicity. pairs = append(pairs, "storageSize", stats.SizeCollection/scale, - "nindexes", stats.CountIndexes, + "nindexes", int64(len(indexes.Indexes)), "totalIndexSize", stats.SizeIndexes/scale, "totalSize", stats.SizeTotal/scale, "scaleFactor", int32(scale), diff --git a/internal/handlers/sqlite/msg_dbstats.go b/internal/handlers/sqlite/msg_dbstats.go index 8743fd097d72..d77ee0aeb152 100644 --- a/internal/handlers/sqlite/msg_dbstats.go +++ b/internal/handlers/sqlite/msg_dbstats.go @@ -66,6 +66,31 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, return nil, lazyerrors.Error(err) } + var nIndexes int64 + + for _, cInfo := range list.Collections { + var c backends.Collection + + c, err = db.Collection(cInfo.Name) + if err != nil { + return nil, lazyerrors.Error(err) + } + + var iList *backends.ListIndexesResult + + iList, err = c.ListIndexes(ctx, new(backends.ListIndexesParams)) + if backends.ErrorCodeIs(err, backends.ErrorCodeCollectionDoesNotExist) { + iList = new(backends.ListIndexesResult) + err = nil + } + + if err != nil { + return nil, lazyerrors.Error(err) + } + + nIndexes += int64(len(iList.Indexes)) + } + stats, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) if backends.ErrorCodeIs(err, backends.ErrorCodeDatabaseDoesNotExist) { stats = new(backends.DatabaseStatsResult) @@ -96,7 +121,7 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, pairs = append(pairs, "dataSize", stats.SizeCollections/scale, "storageSize", stats.SizeCollections/scale, - "indexes", stats.CountIndexes, + "indexes", nIndexes, "indexSize", stats.SizeIndexes/scale, "totalSize", stats.SizeTotal/scale, "scaleFactor", float64(scale), From d0f3022702191b6c1995b74acc9605bf10b8c9eb Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 12:10:35 +0900 Subject: [PATCH 24/27] remove todo --- internal/backends/database.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/backends/database.go b/internal/backends/database.go index ddf2cdb7aa79..8a7254a7f54d 100644 --- a/internal/backends/database.go +++ b/internal/backends/database.go @@ -192,8 +192,6 @@ type DatabaseStatsParams struct { } // DatabaseStatsResult represents the results of Database.Stats method. -// -// TODO https://github.com/FerretDB/FerretDB/issues/2447 type DatabaseStatsResult struct { CountDocuments int64 SizeTotal int64 From 58753c4dce81f1b8a7113c7c2feae38c03d20b6b Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 12:27:23 +0900 Subject: [PATCH 25/27] update comment, undo unwanted change --- integration/commands_administration_compat_test.go | 4 ++-- internal/backends/postgresql/postgresql.go | 10 +++++----- internal/backends/sqlite/sqlite.go | 12 ++++++------ internal/handlers/pg/msg_aggregate.go | 8 ++++---- internal/handlers/pg/msg_collstats.go | 6 +++--- internal/handlers/pg/msg_datasize.go | 2 +- internal/handlers/pg/msg_dbstats.go | 6 +++--- internal/handlers/pg/pgdb/stats.go | 8 ++++---- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/integration/commands_administration_compat_test.go b/integration/commands_administration_compat_test.go index 10b1db51b785..4b01b3e28860 100644 --- a/integration/commands_administration_compat_test.go +++ b/integration/commands_administration_compat_test.go @@ -112,8 +112,8 @@ func TestCommandsAdministrationCompatCollStatsCappedCollection(t *testing.T) { ctx, targetCollection, compatCollection := s.Ctx, s.TargetCollections[0], s.CompatCollections[0] for name, tc := range map[string]struct { //nolint:vet // for readability - sizeInBytes int64 - maxDocuments int64 + sizeInBytes int64 // also sets capped true if it is greater than zero + maxDocuments int64 // maxDocuments is set if sizeInBytes is greater than zero }{ "Size": { sizeInBytes: 1000, diff --git a/internal/backends/postgresql/postgresql.go b/internal/backends/postgresql/postgresql.go index cd4f60c472f4..3fd63e6b55f6 100644 --- a/internal/backends/postgresql/postgresql.go +++ b/internal/backends/postgresql/postgresql.go @@ -32,9 +32,9 @@ import ( // stats represents information about statistics of tables and indexes. type stats struct { - countRows int64 - sizeIndexes int64 - sizeTables int64 + countDocuments int64 + sizeIndexes int64 + sizeTables int64 } // collectionsStats returns statistics about tables and indexes for the given collections. @@ -65,7 +65,7 @@ func collectionsStats(ctx context.Context, p *pgxpool.Pool, dbName string, list args = append(args, c.TableName) } - // The table size is the size used by collection objects. It excludes visibility map, + // The table size is the size used by collection documents. It excludes visibility map, // initialization fork, free space map and TOAST. The `main` `pg_relation_size` is // used, however it is not updated immediately after operation such as DELETE // unless VACUUM is called, ANALYZE does not update pg_relation_size in this case. @@ -90,7 +90,7 @@ func collectionsStats(ctx context.Context, p *pgxpool.Pool, dbName string, list ) row := p.QueryRow(ctx, q, args...) - if err := row.Scan(&s.countRows, &s.sizeTables, &s.sizeIndexes); err != nil { + if err := row.Scan(&s.countDocuments, &s.sizeTables, &s.sizeIndexes); err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/backends/sqlite/sqlite.go b/internal/backends/sqlite/sqlite.go index 80cb37fb5ba6..8e4c4e1c6562 100644 --- a/internal/backends/sqlite/sqlite.go +++ b/internal/backends/sqlite/sqlite.go @@ -42,9 +42,9 @@ import ( // stats represents information about statistics of tables and indexes. type stats struct { - countRows int64 - sizeIndexes int64 - sizeTables int64 + countDocuments int64 + sizeIndexes int64 + sizeTables int64 } // collectionsStats returns statistics about tables and indexes for the given collections. @@ -69,7 +69,7 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect indexes += int64(len(c.Settings.Indexes)) } - // The table size is the size used by collection objects. The `pgsize` of `dbstat` + // The table size is the size used by collection documents. The `pgsize` of `dbstat` // table does not include freelist pages, pointer-map pages, and the lock page. // // If rows are deleted from a page but there are other rows on that same page, @@ -91,7 +91,7 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect return nil, lazyerrors.Error(err) } - // Use number of cells to approximate total row count, + // Use number of cells to approximate total document count, // excluding `internal` and `overflow` pagetype used by SQLite. // See https://www.sqlite.org/dbstat.html and https://www.sqlite.org/fileformat.html. q = fmt.Sprintf(` @@ -101,7 +101,7 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect strings.Join(placeholders, ", "), ) - if err = db.QueryRowContext(ctx, q, args...).Scan(&stats.countRows); err != nil { + if err = db.QueryRowContext(ctx, q, args...).Scan(&stats.countDocuments); err != nil { return nil, lazyerrors.Error(err) } diff --git a/internal/handlers/pg/msg_aggregate.go b/internal/handlers/pg/msg_aggregate.go index e7014745fd99..4f492a6e70a3 100644 --- a/internal/handlers/pg/msg_aggregate.go +++ b/internal/handlers/pg/msg_aggregate.go @@ -428,14 +428,14 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasStorage { var avgObjSize int64 - if collStats.CountDocuments > 0 { - avgObjSize = collStats.SizeCollection / collStats.CountDocuments + if collStats.CountObjects > 0 { + avgObjSize = collStats.SizeCollection / collStats.CountObjects } doc.Set( "storageStats", must.NotFail(types.NewDocument( "size", collStats.SizeTotal, - "count", collStats.CountDocuments, + "count", collStats.CountObjects, "avgObjSize", avgObjSize, "storageSize", collStats.SizeCollection, "freeStorageSize", int64(0), // TODO https://github.com/FerretDB/FerretDB/issues/2342 @@ -453,7 +453,7 @@ func processStagesStats(ctx context.Context, closer *iterator.MultiCloser, p *st if hasCount { doc.Set( - "count", collStats.CountDocuments, + "count", collStats.CountObjects, ) } diff --git a/internal/handlers/pg/msg_collstats.go b/internal/handlers/pg/msg_collstats.go index 0fc59ba0e80d..96d54ad4210c 100644 --- a/internal/handlers/pg/msg_collstats.go +++ b/internal/handlers/pg/msg_collstats.go @@ -84,12 +84,12 @@ func (h *Handler) MsgCollStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMs pairs := []any{ "ns", db + "." + collection, "size", stats.SizeCollection / int64(scale), - "count", stats.CountDocuments, + "count", stats.CountObjects, } // If there are objects in the collection, calculate the average object size. - if stats.CountDocuments > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountDocuments) + if stats.CountObjects > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollection/stats.CountObjects) } // MongoDB uses "numbers" that could be int32 or int64, diff --git a/internal/handlers/pg/msg_datasize.go b/internal/handlers/pg/msg_datasize.go index 7bd6327bb13e..9e8e485faae4 100644 --- a/internal/handlers/pg/msg_datasize.go +++ b/internal/handlers/pg/msg_datasize.go @@ -90,7 +90,7 @@ func (h *Handler) MsgDataSize(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg } pairs = append(pairs, "size", stats.SizeTotal, - "numObjects", stats.CountDocuments, + "numObjects", stats.CountObjects, "millis", int32(elapses.Milliseconds()), "ok", float64(1), ) diff --git a/internal/handlers/pg/msg_dbstats.go b/internal/handlers/pg/msg_dbstats.go index c671fc4b1aa4..8a484dec9145 100644 --- a/internal/handlers/pg/msg_dbstats.go +++ b/internal/handlers/pg/msg_dbstats.go @@ -70,11 +70,11 @@ func (h *Handler) MsgDBStats(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg, "collections", stats.CountCollections, // TODO https://github.com/FerretDB/FerretDB/issues/176 "views", int32(0), - "objects", stats.CountDocuments, + "objects", stats.CountObjects, } - if stats.CountDocuments > 0 { - pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountDocuments) + if stats.CountObjects > 0 { + pairs = append(pairs, "avgObjSize", stats.SizeCollections/stats.CountObjects) } // MongoDB uses "numbers" that could be int32 or int64, diff --git a/internal/handlers/pg/pgdb/stats.go b/internal/handlers/pg/pgdb/stats.go index 8fac3dddcfd9..3a9039298767 100644 --- a/internal/handlers/pg/pgdb/stats.go +++ b/internal/handlers/pg/pgdb/stats.go @@ -38,7 +38,7 @@ type ServerStats struct { // TODO https://github.com/FerretDB/FerretDB/issues/2447 type DBStats struct { CountCollections int64 - CountDocuments int64 + CountObjects int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 @@ -53,7 +53,7 @@ type DBStats struct { // Include more data. // TODO https://github.com/FerretDB/FerretDB/issues/2447 type CollStats struct { - CountDocuments int64 + CountObjects int64 CountIndexes int64 SizeTotal int64 SizeIndexes int64 @@ -131,7 +131,7 @@ func CalculateDBStats(ctx context.Context, tx pgx.Tx, db string) (*DBStats, erro row = tx.QueryRow(ctx, sql, args...) if err := row.Scan( - &res.CountCollections, &res.CountIndexes, &res.CountDocuments, &res.SizeCollections, &res.SizeIndexes, + &res.CountCollections, &res.CountIndexes, &res.CountObjects, &res.SizeCollections, &res.SizeIndexes, ); err != nil { return nil, lazyerrors.Error(err) } @@ -175,7 +175,7 @@ func CalculateCollStats(ctx context.Context, tx pgx.Tx, db, collection string) ( ) row := tx.QueryRow(ctx, sql) - if err := row.Scan(&res.CountDocuments, &res.SizeTotal, &res.SizeCollection, &res.SizeIndexes); err != nil { + if err := row.Scan(&res.CountObjects, &res.SizeTotal, &res.SizeCollection, &res.SizeIndexes); err != nil { return nil, lazyerrors.Error(err) } From cd705181ba5a07b4bcb7fe9c12ce42e5e6927543 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 12:30:07 +0900 Subject: [PATCH 26/27] lint --- internal/backends/postgresql/collection.go | 2 +- internal/backends/postgresql/database.go | 2 +- internal/backends/sqlite/collection.go | 2 +- internal/backends/sqlite/database.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/backends/postgresql/collection.go b/internal/backends/postgresql/collection.go index ec0096b2b033..a032d622afad 100644 --- a/internal/backends/postgresql/collection.go +++ b/internal/backends/postgresql/collection.go @@ -347,7 +347,7 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats } return &backends.CollectionStatsResult{ - CountDocuments: stats.countRows, + CountDocuments: stats.countDocuments, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, SizeCollection: stats.sizeTables, diff --git a/internal/backends/postgresql/database.go b/internal/backends/postgresql/database.go index d9d7becdfbba..36ffb5f1fe22 100644 --- a/internal/backends/postgresql/database.go +++ b/internal/backends/postgresql/database.go @@ -172,7 +172,7 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar } return &backends.DatabaseStatsResult{ - CountDocuments: stats.countRows, + CountDocuments: stats.countDocuments, SizeTotal: *schemaSize, SizeIndexes: stats.sizeIndexes, SizeCollections: stats.sizeTables, diff --git a/internal/backends/sqlite/collection.go b/internal/backends/sqlite/collection.go index 5a2cb7f8708e..29578ee13274 100644 --- a/internal/backends/sqlite/collection.go +++ b/internal/backends/sqlite/collection.go @@ -321,7 +321,7 @@ func (c *collection) Stats(ctx context.Context, params *backends.CollectionStats } return &backends.CollectionStatsResult{ - CountDocuments: stats.countRows, + CountDocuments: stats.countDocuments, SizeTotal: stats.sizeTables + stats.sizeIndexes, SizeIndexes: stats.sizeIndexes, SizeCollection: stats.sizeTables, diff --git a/internal/backends/sqlite/database.go b/internal/backends/sqlite/database.go index b44da4a5fc07..b14fd82a5e12 100644 --- a/internal/backends/sqlite/database.go +++ b/internal/backends/sqlite/database.go @@ -148,7 +148,7 @@ func (db *database) Stats(ctx context.Context, params *backends.DatabaseStatsPar } return &backends.DatabaseStatsResult{ - CountDocuments: stats.countRows, + CountDocuments: stats.countDocuments, SizeTotal: totalSize, SizeIndexes: stats.sizeIndexes, SizeCollections: stats.sizeTables, From f85bc753af815b92e2d836518e5f67c5b3faf415 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Mon, 16 Oct 2023 12:33:29 +0900 Subject: [PATCH 27/27] add missed commit --- integration/commands_administration_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/integration/commands_administration_test.go b/integration/commands_administration_test.go index 796dda5f4d78..968a5bcda088 100644 --- a/integration/commands_administration_test.go +++ b/integration/commands_administration_test.go @@ -700,8 +700,6 @@ func TestCommandsAdministrationCollStatsEmpty(t *testing.T) { } func TestCommandsAdministrationCollStats(t *testing.T) { - setup.SkipForPostgreSQL(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) @@ -745,8 +743,6 @@ func TestCommandsAdministrationCollStats(t *testing.T) { } func TestCommandsAdministrationCollStatsWithScale(t *testing.T) { - setup.SkipForPostgreSQL(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() ctx, collection := setup.Setup(t, shareddata.DocumentsStrings) @@ -796,8 +792,6 @@ func TestCommandsAdministrationCollStatsCount(t *testing.T) { } func TestCommandsAdministrationDataSize(t *testing.T) { - setup.SkipForPostgreSQL(t, "https://github.com/FerretDB/FerretDB/issues/3521") - t.Parallel() t.Run("Existing", func(t *testing.T) {