Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace document slices with iterators #2730

Merged
merged 19 commits into from
Jun 2, 2023
Merged

Replace document slices with iterators #2730

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
9ebcea3
wip
noisersup May 29, 2023
963e2a4
wip
noisersup May 29, 2023
8ccbf0e
tigris
noisersup May 29, 2023
d5354bb
Merge branch 'main' into more-iterators-2723
noisersup May 30, 2023
8283d9c
wip
noisersup May 29, 2023
3745570
wip
noisersup May 30, 2023
798de56
wip
noisersup May 30, 2023
03fa375
wip
noisersup May 31, 2023
9e86c5b
wip
noisersup May 31, 2023
ede5dc8
wip
noisersup May 31, 2023
50159a5
wip
noisersup Jun 1, 2023
9ee12de
Merge remote-tracking branch 'upstream/main' into more-iterators-2723
noisersup Jun 1, 2023
c9edd10
wip
noisersup Jun 1, 2023
63197b4
wip
noisersup Jun 1, 2023
d08ec40
Merge branch 'main' into more-iterators-2723
mergify[bot] Jun 2, 2023
445aa88
Merge branch 'main' into more-iterators-2723
mergify[bot] Jun 2, 2023
b0a51ff
Merge branch 'main' into more-iterators-2723
mergify[bot] Jun 2, 2023
ba79eed
Merge branch 'main' into more-iterators-2723
mergify[bot] Jun 2, 2023
2c3b544
Merge branch 'main' into more-iterators-2723
mergify[bot] Jun 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
wip
  • Loading branch information
@noisersup
noisersup committed May 30, 2023
commit 798de5698ac32cb8122297712384bf3adfd9bbc4
17 changes: 3 additions & 14 deletions internal/handlers/common/distinct.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"github.com/FerretDB/FerretDB/internal/handlers/commonerrors"
"github.com/FerretDB/FerretDB/internal/handlers/commonparams"
"github.com/FerretDB/FerretDB/internal/types"
"github.com/FerretDB/FerretDB/internal/util/iterator"
"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
)

Expand Down Expand Up @@ -66,20 +65,10 @@ func GetDistinctParams(document *types.Document, l *zap.Logger) (*DistinctParams
//
// If the key is found in the document, and the value is an array, each element of the array is added to the result.
// Otherwise, the value itself is added to the result.
func FilterDistinctValues(iter types.DocumentsIterator, key string) (*types.Array, error) {
distinct := types.MakeArray(0)
defer iter.Close()

for {
_, doc, err := iter.Next()
if err != nil {
if err == iterator.ErrIteratorDone {
break
}

return nil, err
}
func FilterDistinctValues(docs []*types.Document, key string) (*types.Array, error) {
distinct := types.MakeArray(len(docs))

for _, doc := range docs {
// docsAtSuffix contains all documents exist at the suffix key.
docsAtSuffix := []*types.Document{doc}
suffix := key
Expand Down
19 changes: 4 additions & 15 deletions internal/handlers/tigris/msg_distinct.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"github.com/FerretDB/FerretDB/internal/handlers/common"
"github.com/FerretDB/FerretDB/internal/handlers/tigris/tigrisdb"
"github.com/FerretDB/FerretDB/internal/types"
"github.com/FerretDB/FerretDB/internal/util/iterator"
"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/internal/wire"
Expand All @@ -46,25 +45,15 @@ func (h *Handler) MsgDistinct(ctx context.Context, msg *wire.OpMsg) (*wire.OpMsg
qp := tigrisdb.QueryParams{
DB: dp.DB,
Collection: dp.Collection,
Filter: dp.Filter,
}

if !h.DisableFilterPushdown {
qp.Filter = dp.Filter
}

fp := &fetchParams{dbPool, &qp, h.DisableFilterPushdown}

iter, err := fp.dbPool.QueryDocuments(ctx, fp.qp)
resDocs, err := fetchAndFilterDocs(ctx, &fetchParams{dbPool, &qp, h.DisableFilterPushdown})
if err != nil {
return nil, lazyerrors.Error(err)
return nil, err
}

closer := iterator.NewMultiCloser(iter)
defer closer.Close()

iter = common.FilterIterator(iter, closer, qp.Filter)

distinct, err := common.FilterDistinctValues(iter, dp.Key)
distinct, err := common.FilterDistinctValues(resDocs, dp.Key)
if err != nil {
return nil, err
}
Expand Down