Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make batch sizes configurable #4149

Merged
merged 22 commits into from
Mar 19, 2024
Merged

Make batch sizes configurable #4149

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1bf9901
add batchsize support for Handler
kropidlowsky Feb 27, 2024
b228093
WIP
kropidlowsky Mar 4, 2024
309ccbf
WIP
kropidlowsky Mar 4, 2024
d43898b
fix postgres TestDatabaseStats
kropidlowsky Mar 4, 2024
b277e01
fix sqlite TestCappedCollectionInsertAllQueryExplain
kropidlowsky Mar 4, 2024
2a6219a
fix sqlite TestDatabaseStatsFreeStorage
kropidlowsky Mar 4, 2024
2c51cab
WIP
kropidlowsky Mar 5, 2024
a421052
wip
kropidlowsky Mar 5, 2024
d5f1052
Merge branch 'main' into batch_sizes
kropidlowsky Mar 6, 2024
4700ca4
Merge branch 'main' into batch_sizes
AlekSi Mar 6, 2024
d94737e
Merge branch 'main' into batch_sizes
AlekSi Mar 7, 2024
cae4c98
add panic for `batchSize` at `InsertAll`
kropidlowsky Mar 11, 2024
8913109
Merge branch 'main' into batch_sizes
kropidlowsky Mar 11, 2024
039ebeb
Merge branch 'main' into batch_sizes
kropidlowsky Mar 12, 2024
5092b59
add `batch-size` for integration
kropidlowsky Mar 13, 2024
8099fc2
improove `batch-size` panic message
kropidlowsky Mar 13, 2024
a6f8dfd
Merge branch 'main' into batch_sizes
kropidlowsky Mar 13, 2024
75b8ae6
correct flag help info
kropidlowsky Mar 14, 2024
a9cbcc4
Merge branch 'main' into batch_sizes
kropidlowsky Mar 14, 2024
fb1d058
Merge branch 'main' into batch_sizes
AlekSi Mar 19, 2024
7aea53b
Improve flag description
AlekSi Mar 19, 2024
0a86ac7
Simplify
AlekSi Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
add batchsize support for Handler
  • Loading branch information
@kropidlowsky
kropidlowsky committed Feb 27, 2024
commit 1bf99019b1052eea6c87d326a805057e0080e86d
2 changes: 2 additions & 0 deletions cmd/ferretdb/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ var cli struct {
} `embed:"" prefix:"capped-cleanup-"`

EnableNewAuth bool `default:"false" help:"Experimental: enable new authentication."`
BatchSize int `default:"100" help:"number of maximum size of query parameters"`

Telemetry struct {
URL string `default:"https://beacon.ferretdb.com/" help:"Telemetry: reporting URL."`
Expand Down Expand Up @@ -412,6 +413,7 @@ func run() {
CappedCleanupInterval: cli.Test.CappedCleanup.Interval,
CappedCleanupPercentage: cli.Test.CappedCleanup.Percentage,
EnableNewAuth: cli.Test.EnableNewAuth,
BatchSize: cli.Test.BatchSize,
},
})
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions internal/handler/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ type NewOpts struct {
CappedCleanupInterval time.Duration
CappedCleanupPercentage uint8
EnableNewAuth bool
BatchSize int
}

// New returns a new handler.
Expand Down
8 changes: 3 additions & 5 deletions internal/handler/msg_insert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,11 @@ func (h *Handler) MsgInsert(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg,

var done bool
for !done {
// TODO https://github.com/FerretDB/FerretDB/issues/3708
const batchSize = 1000

docs := make([]*types.Document, 0, batchSize)
docsIndexes := make([]int, 0, batchSize)
docs := make([]*types.Document, 0, h.BatchSize)
docsIndexes := make([]int, 0, h.BatchSize)

for j := 0; j < batchSize; j++ {
for j := 0; j < h.BatchSize; j++ {
var i int
var d any

Expand Down
1 change: 1 addition & 0 deletions internal/handler/registry/hana.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func init() {
CappedCleanupPercentage: opts.CappedCleanupPercentage,
CappedCleanupInterval: opts.CappedCleanupInterval,
EnableNewAuth: opts.EnableNewAuth,
BatchSize: opts.BatchSize
}

h, err := handler.New(handlerOpts)
Expand Down
1 change: 1 addition & 0 deletions internal/handler/registry/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func init() {
CappedCleanupPercentage: opts.CappedCleanupPercentage,
CappedCleanupInterval: opts.CappedCleanupInterval,
EnableNewAuth: opts.EnableNewAuth,
BatchSize: opts.BatchSize,
}

h, err := handler.New(handlerOpts)
Expand Down
1 change: 1 addition & 0 deletions internal/handler/registry/postgresql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func init() {
CappedCleanupPercentage: opts.CappedCleanupPercentage,
CappedCleanupInterval: opts.CappedCleanupInterval,
EnableNewAuth: opts.EnableNewAuth,
BatchSize: opts.BatchSize,
}

h, err := handler.New(handlerOpts)
Expand Down
1 change: 1 addition & 0 deletions internal/handler/registry/registry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ type TestOpts struct {
CappedCleanupInterval time.Duration
CappedCleanupPercentage uint8
EnableNewAuth bool
BatchSize int
_ struct{} // prevent unkeyed literals
}

Expand Down
1 change: 1 addition & 0 deletions internal/handler/registry/sqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func init() {
CappedCleanupPercentage: opts.CappedCleanupPercentage,
CappedCleanupInterval: opts.CappedCleanupInterval,
EnableNewAuth: opts.EnableNewAuth,
BatchSize: opts.BatchSize,
}

h, err := handler.New(handlerOpts)
Expand Down