From 13ebdf7d57ffb5c565e782807493431c78dd9769 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 6 Oct 2023 19:13:12 +0900 Subject: [PATCH 1/3] init --- internal/backends/collection_test.go | 2 +- internal/backends/postgresql/database_test.go | 2 +- internal/backends/postgresql/postgresql.go | 15 ++++++++++++++- internal/backends/sqlite/sqlite.go | 10 ++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index 2817de2a4e67..aa713b6842b9 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -211,7 +211,7 @@ func TestCollectionStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Less(t, res.SizeTotal, dbStatsRes.SizeTotal) require.NotZero(t, res.SizeCollection) - require.Less(t, res.SizeCollection, dbStatsRes.SizeCollections) + require.LessOrEqual(t, res.SizeCollection, dbStatsRes.SizeCollections) require.Equal(t, res.CountObjects, int64(1)) // TODO https://github.com/FerretDB/FerretDB/issues/3394 // require.NotZero(t, res.CountIndexes) diff --git a/internal/backends/postgresql/database_test.go b/internal/backends/postgresql/database_test.go index 66eb3cdd8b63..56dd09eb3726 100644 --- a/internal/backends/postgresql/database_test.go +++ b/internal/backends/postgresql/database_test.go @@ -69,7 +69,7 @@ func TestDatabaseStats(t *testing.T) { require.NoError(t, err) require.NotZero(t, res.SizeTotal) require.Equal(t, res.CountCollections, int64(len(cNames))) - require.NotZero(t, res.SizeCollections) + require.Zero(t, res.SizeCollections) require.Zero(t, res.CountObjects) // TODO https://github.com/FerretDB/FerretDB/issues/3394 // require.NotZero(t, res.CountIndexes) diff --git a/internal/backends/postgresql/postgresql.go b/internal/backends/postgresql/postgresql.go index aa4a3737a903..d7065cb32d4e 100644 --- a/internal/backends/postgresql/postgresql.go +++ b/internal/backends/postgresql/postgresql.go @@ -65,10 +65,23 @@ func collectionsStats(ctx context.Context, p *pgxpool.Pool, dbName string, list // TODO https://github.com/FerretDB/FerretDB/issues/3394 s.countIndexes = 0 + // The table size is the size used by collection objects. It excludes visibility map, + // initialization fork, free space map and TOAST. The `main` `pg_relation_size` is + // used, however it is not updated immediately after operation such as DELETE + // unless VACUUM is called, ANALYZE does not update pg_relation_size in this case. + // + // The smallest difference in size that `pg_relation_size` reports appears to be 8KB. + // Because of that inserting or deleting a single small object may not change the size. + // + // See also https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE, + // visibility map https://www.postgresql.org/docs/current/storage-vm.html, + // initialization fork https://www.postgresql.org/docs/current/storage-init.html, + // free space map https://www.postgresql.org/docs/current/storage-fsm.html and + // TOAST https://www.postgresql.org/docs/current/storage-toast.html. q = fmt.Sprintf(` SELECT COALESCE(SUM(c.reltuples), 0), - COALESCE(SUM(pg_table_size(c.oid)), 0), + COALESCE(SUM(pg_relation_size(c.oid,'main')), 0), COALESCE(SUM(pg_indexes_size(c.oid)), 0) FROM pg_tables AS t LEFT JOIN pg_class AS c ON c.relname = t.tablename AND c.relnamespace = quote_ident(t.schemaname)::regnamespace diff --git a/internal/backends/sqlite/sqlite.go b/internal/backends/sqlite/sqlite.go index 2471e8c77720..75e26ed92358 100644 --- a/internal/backends/sqlite/sqlite.go +++ b/internal/backends/sqlite/sqlite.go @@ -71,6 +71,16 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect indexes += int64(len(c.Settings.Indexes)) } + // The table size is the size used by collection objects. The `pgsize` of `dbstat` table + // does not include freelist pages, pointer-map pages, and the lock page. + // + // If rows are deleted from a page but there are other rows on that same page, + // the page won't be moved to freelist pages. + // + // The smallest difference in size that `pgsize` reports appears to be 4KB. + // Because of that inserting or deleting a single small object may not change the size. + // + // See https://www.sqlite.org/dbstat.html and https://www.sqlite.org/fileformat.html. q = fmt.Sprintf(` SELECT SUM(pgsize) FROM dbstat From 91e709316262df16b3c949f59a8b121cbf6c1cbd Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Fri, 6 Oct 2023 19:32:14 +0900 Subject: [PATCH 2/3] insert in each collection --- internal/backends/collection_test.go | 17 +++++++++-------- internal/backends/sqlite/database_test.go | 2 ++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/backends/collection_test.go b/internal/backends/collection_test.go index aa713b6842b9..e3bbeed253cb 100644 --- a/internal/backends/collection_test.go +++ b/internal/backends/collection_test.go @@ -189,19 +189,20 @@ func TestCollectionStats(t *testing.T) { db, err := b.Database(dbName) require.NoError(t, err) + var c backends.Collection cNames := []string{"collectionOne", "collectionTwo"} for _, cName := range cNames { err = db.CreateCollection(ctx, &backends.CreateCollectionParams{Name: cName}) require.NoError(t, err) - } - c, err := db.Collection(cNames[0]) - require.NoError(t, err) + c, err = db.Collection(cName) + require.NoError(t, err) - _, err = c.InsertAll(ctx, &backends.InsertAllParams{ - Docs: []*types.Document{must.NotFail(types.NewDocument("_id", types.NewObjectID()))}, - }) - require.NoError(t, err) + _, err = c.InsertAll(ctx, &backends.InsertAllParams{ + Docs: []*types.Document{must.NotFail(types.NewDocument("_id", types.NewObjectID()))}, + }) + require.NoError(t, err) + } dbStatsRes, err := db.Stats(ctx, new(backends.DatabaseStatsParams)) require.NoError(t, err) @@ -211,7 +212,7 @@ func TestCollectionStats(t *testing.T) { require.NotZero(t, res.SizeTotal) require.Less(t, res.SizeTotal, dbStatsRes.SizeTotal) require.NotZero(t, res.SizeCollection) - require.LessOrEqual(t, res.SizeCollection, dbStatsRes.SizeCollections) + require.Less(t, res.SizeCollection, dbStatsRes.SizeCollections) require.Equal(t, res.CountObjects, int64(1)) // TODO https://github.com/FerretDB/FerretDB/issues/3394 // require.NotZero(t, res.CountIndexes) diff --git a/internal/backends/sqlite/database_test.go b/internal/backends/sqlite/database_test.go index 3d5ebcdcf99e..3590403a468e 100644 --- a/internal/backends/sqlite/database_test.go +++ b/internal/backends/sqlite/database_test.go @@ -20,6 +20,8 @@ import ( "github.com/stretchr/testify/require" "github.com/FerretDB/FerretDB/internal/backends" + "github.com/FerretDB/FerretDB/internal/types" + "github.com/FerretDB/FerretDB/internal/util/must" "github.com/FerretDB/FerretDB/internal/util/state" "github.com/FerretDB/FerretDB/internal/util/testutil" ) From 4ccee109048d7a435eb538db483c383e76055707 Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 10 Oct 2023 11:16:36 +0900 Subject: [PATCH 3/3] fix merge --- internal/backends/postgresql/postgresql.go | 8 +------- internal/backends/sqlite/sqlite.go | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/internal/backends/postgresql/postgresql.go b/internal/backends/postgresql/postgresql.go index d9bc1db71779..54b869ca8e9b 100644 --- a/internal/backends/postgresql/postgresql.go +++ b/internal/backends/postgresql/postgresql.go @@ -85,15 +85,9 @@ func collectionsStats(ctx context.Context, p *pgxpool.Pool, dbName string, list // TOAST https://www.postgresql.org/docs/current/storage-toast.html. q = fmt.Sprintf(` SELECT -<<<<<<< HEAD - COALESCE(SUM(c.reltuples), 0), - COALESCE(SUM(pg_relation_size(c.oid,'main')), 0), - COALESCE(SUM(pg_indexes_size(c.oid)), 0) -======= COALESCE(SUM(c.reltuples), 0), - COALESCE(SUM(pg_table_size(c.oid)), 0), + COALESCE(SUM(pg_relation_size(c.oid,'main')), 0), COALESCE(SUM(pg_indexes_size(c.oid)), 0) ->>>>>>> main FROM pg_tables AS t LEFT JOIN pg_class AS c ON c.relname = t.tablename AND c.relnamespace = quote_ident(t.schemaname)::regnamespace WHERE t.schemaname = $1 AND t.tablename IN (%s)`, diff --git a/internal/backends/sqlite/sqlite.go b/internal/backends/sqlite/sqlite.go index f90011a80577..473f19a19440 100644 --- a/internal/backends/sqlite/sqlite.go +++ b/internal/backends/sqlite/sqlite.go @@ -70,8 +70,8 @@ func collectionsStats(ctx context.Context, db *fsql.DB, list []*metadata.Collect indexes += int64(len(c.Settings.Indexes)) } - // The table size is the size used by collection objects. The `pgsize` of `dbstat` table - // does not include freelist pages, pointer-map pages, and the lock page. + // The table size is the size used by collection objects. The `pgsize` of `dbstat` + // table does not include freelist pages, pointer-map pages, and the lock page. // // If rows are deleted from a page but there are other rows on that same page, // the page won't be moved to freelist pages.