Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support create for capped collections #3614

Merged
merged 35 commits into from
Oct 24, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
09c7c21
wip
rumyantseva Oct 19, 2023
acfc3eb
wip
rumyantseva Oct 19, 2023
36cafbf
wip
rumyantseva Oct 19, 2023
6b03ca0
wip
rumyantseva Oct 19, 2023
2b288d5
wip
rumyantseva Oct 19, 2023
8f57936
wip
rumyantseva Oct 19, 2023
a1ac944
Merge branch 'main' into issue-3458-create
Oct 19, 2023
53fe697
wip
rumyantseva Oct 19, 2023
0155136
wip
rumyantseva Oct 19, 2023
c825d42
Update internal/backends/sqlite/metadata/registry.go
Oct 20, 2023
8d24039
wip
rumyantseva Oct 20, 2023
b5ccc14
wip
rumyantseva Oct 20, 2023
ce23e40
wip
rumyantseva Oct 20, 2023
578bade
wip
rumyantseva Oct 20, 2023
7b49f0b
Update integration/commands_administration_compat_test.go
Oct 20, 2023
4dfab8c
Merge branch 'main' into issue-3458-create
Oct 20, 2023
b8eb619
wip
rumyantseva Oct 20, 2023
1a9299c
Merge branch 'issue-3458-create' of https://github.com/rumyantseva/Fe…
rumyantseva Oct 20, 2023
bf1d2ab
wip
rumyantseva Oct 20, 2023
799d049
wip
rumyantseva Oct 20, 2023
278972d
wip
rumyantseva Oct 20, 2023
e2eeae6
Merge branch 'main' into issue-3458-create
Oct 20, 2023
04b06b6
wip
rumyantseva Oct 20, 2023
132cf75
Better error
rumyantseva Oct 23, 2023
3eed980
Merge branch 'main' into issue-3458-create
AlekSi Oct 23, 2023
907c4af
wip
rumyantseva Oct 23, 2023
d269272
Merge branch 'issue-3458-create' of https://github.com/rumyantseva/Fe…
rumyantseva Oct 23, 2023
ec8755e
wip
rumyantseva Oct 23, 2023
48dc636
Update internal/backends/postgresql/metadata/metadata.go
Oct 23, 2023
f75012a
Update internal/backends/sqlite/metadata/settings.go
Oct 23, 2023
562d2cd
Update integration/commands_administration_compat_test.go
Oct 23, 2023
aef0773
wip
rumyantseva Oct 24, 2023
afc5470
refactor
chilagrow Oct 24, 2023
eeaa953
cleanup
chilagrow Oct 24, 2023
8d836d7
Merge branch 'main' into issue-3458-create
AlekSi Oct 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
wip
  • Loading branch information
rumyantseva committed Oct 20, 2023
commit 799d0493df5cc656caf802a81ac0dde045da6e5a
9 changes: 8 additions & 1 deletion internal/backends/sqlite/collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,14 @@ func (c *collection) Query(ctx context.Context, params *backends.QueryParams) (*

// InsertAll implements backends.Collection interface.
func (c *collection) InsertAll(ctx context.Context, params *backends.InsertAllParams) (*backends.InsertAllResult, error) {
if _, err := c.r.CollectionCreate(ctx, c.dbName, c.name, nil); err != nil {
mparams := metadata.CollectionCreateParams{
DBName: c.dbName,
Name: c.name,
CappedSize: 0,
CappedDocuments: 0,
}

if _, err := c.r.CollectionCreate(ctx, &mparams); err != nil {
return nil, lazyerrors.Error(err)
}

Expand Down
23 changes: 9 additions & 14 deletions internal/backends/sqlite/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,9 @@ func (db *database) ListCollections(ctx context.Context, params *backends.ListCo
res := make([]backends.CollectionInfo, len(list))
for i, c := range list {
res[i] = backends.CollectionInfo{
Name: c.Name,
}

if c.Settings.Capped != nil {
res[i].CappedSize = c.Settings.Capped.Size
res[i].CappedDocuments = c.Settings.Capped.Docs
Name: c.Name,
CappedSize: c.Settings.CappedSize,
CappedDocuments: c.Settings.CappedDocuments,
}
}

Expand All @@ -69,16 +66,14 @@ func (db *database) ListCollections(ctx context.Context, params *backends.ListCo

// CreateCollection implements backends.Database interface.
func (db *database) CreateCollection(ctx context.Context, params *backends.CreateCollectionParams) error {
var capped *metadata.Capped

if params.CappedSize != 0 {
capped = &metadata.Capped{
Size: params.CappedSize,
Docs: params.CappedDocuments,
}
mparams := metadata.CollectionCreateParams{
DBName: db.name,
Name: params.Name,
CappedSize: params.CappedSize,
CappedDocuments: params.CappedDocuments,
}

created, err := db.r.CollectionCreate(ctx, db.name, params.Name, capped)
created, err := db.r.CollectionCreate(ctx, &mparams)
if err != nil {
return lazyerrors.Error(err)
}
Expand Down
24 changes: 14 additions & 10 deletions internal/backends/sqlite/metadata/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,10 @@ func (r *Registry) CollectionList(ctx context.Context, dbName string) ([]*Collec
return res, nil
}

// CollectionCreteParams contains parameters for CollectionCreate.
type CollectionCreteParams struct {
// CollectionCreateParams contains parameters for CollectionCreate.
type CollectionCreateParams struct {
DBName string
CollectionName string
Name string
CappedSize int64
CappedDocuments int64
}
Expand All @@ -242,13 +242,13 @@ type CollectionCreteParams struct {
//
// Returned boolean value indicates whether the collection was created.
// If collection already exists, (false, nil) is returned.
func (r *Registry) CollectionCreate(ctx context.Context, dbName, collectionName string, capped *Capped) (bool, error) {
func (r *Registry) CollectionCreate(ctx context.Context, params *CollectionCreateParams) (bool, error) {
defer observability.FuncCall(ctx)()

r.rw.Lock()
defer r.rw.Unlock()

return r.collectionCreate(ctx, dbName, collectionName, capped)
return r.collectionCreate(ctx, params)
}

// collectionCreate creates a collection in the database.
Expand All @@ -258,9 +258,11 @@ func (r *Registry) CollectionCreate(ctx context.Context, dbName, collectionName
// If collection already exists, (false, nil) is returned.
//
// It does not hold the lock.
func (r *Registry) collectionCreate(ctx context.Context, dbName, collectionName string, capped *Capped) (bool, error) {
func (r *Registry) collectionCreate(ctx context.Context, params *CollectionCreateParams) (bool, error) {
defer observability.FuncCall(ctx)()

dbName, collectionName := params.DBName, params.Name

db, err := r.databaseGetOrCreate(ctx, dbName)
if err != nil {
return false, lazyerrors.Error(err)
Expand Down Expand Up @@ -294,7 +296,7 @@ func (r *Registry) collectionCreate(ctx context.Context, dbName, collectionName

q := fmt.Sprintf("CREATE TABLE %q (", tableName)

if capped != nil {
if params.CappedSize > 0 {
q += fmt.Sprintf("%s INTEGER PRIMARY KEY, ", RecordIDColumn)
}

Expand All @@ -316,10 +318,12 @@ func (r *Registry) collectionCreate(ctx context.Context, dbName, collectionName
r.colls[dbName][collectionName] = &Collection{
Name: collectionName,
TableName: tableName,
Settings: Settings{
CappedSize: params.CappedSize,
CappedDocuments: params.CappedDocuments,
},
}

r.colls[dbName][collectionName].Settings.Capped = capped

err = r.indexesCreate(ctx, dbName, collectionName, []IndexInfo{{
Name: backends.DefaultIndexName,
Key: []IndexKeyPair{{Field: "_id"}},
Expand Down Expand Up @@ -462,7 +466,7 @@ func (r *Registry) IndexesCreate(ctx context.Context, dbName, collectionName str
func (r *Registry) indexesCreate(ctx context.Context, dbName, collectionName string, indexes []IndexInfo) error {
defer observability.FuncCall(ctx)()

_, err := r.collectionCreate(ctx, dbName, collectionName, nil)
_, err := r.collectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
if err != nil {
return lazyerrors.Error(err)
}
Expand Down
12 changes: 6 additions & 6 deletions internal/backends/sqlite/metadata/registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ func testCollection(t *testing.T, ctx context.Context, r *Registry, db *fsql.DB,
c := r.CollectionGet(ctx, dbName, collectionName)
require.Nil(t, c)

created, err := r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err := r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
require.True(t, created)

created, err = r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err = r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
require.False(t, created)

Expand Down Expand Up @@ -190,13 +190,13 @@ func TestCreateSameStress(t *testing.T) {
ready <- struct{}{}
<-start

created, err := r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err := r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
if created {
createdTotal.Add(1)
}

created, err = r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err = r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
require.False(t, created)

Expand Down Expand Up @@ -255,7 +255,7 @@ func TestDropSameStress(t *testing.T) {

collectionName := "collection"

created, err := r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err := r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
require.True(t, created)

Expand Down Expand Up @@ -321,7 +321,7 @@ func TestCreateDropSameStress(t *testing.T) {
<-start

if id%2 == 0 {
created, err := r.CollectionCreate(ctx, dbName, collectionName, nil)
created, err := r.CollectionCreate(ctx, &CollectionCreateParams{dbName, collectionName, 0, 0})
require.NoError(t, err)
if created {
createdTotal.Add(1)
Expand Down
26 changes: 7 additions & 19 deletions internal/backends/sqlite/metadata/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,9 @@ import (

// Settings represents collection settings.
type Settings struct {
Indexes []IndexInfo `json:"indexes"`
Capped *Capped `json:"capped,omitempty"` // nil if not capped
}

// Capped represents capped collection settings.
type Capped struct {
Size int64 `json:"size"`
Docs int64 `json:"docs"` // 0 if not set
Indexes []IndexInfo `json:"indexes"`
CappedSize int64 `json:"cappedSize"` // 0 if not capped
CappedDocuments int64 `json:"cappedDocs"` // 0 if the limit is not set
}

// IndexInfo represents information about a single index.
Expand Down Expand Up @@ -60,18 +55,11 @@ func (s Settings) deepCopy() Settings {
}
}

settings := Settings{
Indexes: indexes,
}

if s.Capped != nil {
settings.Capped = &Capped{
Size: s.Capped.Size,
Docs: s.Capped.Docs,
}
return Settings{
Indexes: indexes,
CappedSize: s.CappedSize,
CappedDocuments: s.CappedDocuments,
}

return settings
}

// Value implements driver.Valuer interface.
Expand Down
3 changes: 2 additions & 1 deletion internal/backends/sqlite/sqlite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ func TestCollectionsStats(t *testing.T) {
colls := make([]*metadata.Collection, len(cNames))

for i, cName := range cNames {
_, err = r.CollectionCreate(ctx, dbName, cName, nil)
params := &metadata.CollectionCreateParams{DBName: dbName, Name: cName}
_, err = r.CollectionCreate(ctx, params)
require.NoError(t, err)
colls[i] = r.CollectionGet(ctx, dbName, cName)
}
Expand Down
Loading