Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

partition ranges, covering indexes, smarter iterators #1116

Merged
merged 13 commits into from
Dec 16, 2020
Merged

partition ranges, covering indexes, smarter iterators #1116

Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
be2c221
covering indexes
Dec 11, 2020
52108ae
ignore this venv dir that I got from sysbench
Dec 11, 2020
9110c9c
skip some broken query explanation tests. Zach is going to look into…
Dec 11, 2020
38e9b92
merge partition and covering index changes
Dec 12, 2020
7bef098
test new map iterator for indexes
Dec 12, 2020
30b3485
index optimizations and fixes
Dec 15, 2020
628ec20
perf comparison script
Dec 15, 2020
1c77b54
merged master, resolved conflicts
andy-wm-arthur Dec 15, 2020
64f41d9
fix copyright headers, repofmt
andy-wm-arthur Dec 15, 2020
12d95b8
incensitive column name checks
Dec 16, 2020
1170757
pr feedback
Dec 16, 2020
5e7c137
PR feedback
Dec 16, 2020
33111c6
pr feedback
Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
pr feedback
  • Loading branch information
Brian Hendriks committed Dec 16, 2020
commit 117075715e515dda4ddf99082ae2d7fb511fe9f1
28 changes: 26 additions & 2 deletions go/libraries/doltcore/sqle/dolt_map_iter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package sqle

import (
"context"
"errors"
"io"

"github.com/dolthub/go-mysql-server/sql"
Expand All @@ -24,12 +25,18 @@ import (
"github.com/dolthub/dolt/go/store/types"
)

// KVToSqlRowConverter takes noms types.Value key value pairs and converts them directly to a sql.Row directly. It
bheni marked this conversation as resolved.
Show resolved Hide resolved
// can be configured to only process a portion of the columns and map columens to desired output columns.
bheni marked this conversation as resolved.
Show resolved Hide resolved
type KVToSqlRowConverter struct {
tagToSqlColIdx map[uint64]int
cols []schema.Column
// rowSize is the number of columns in the output row. This may be bigger than the number of columns being converted,
// but not less. When rowSize is bigger than the number of columns being processed that means that some of the columns
// in the output row will be filled with nils
rowSize int
}

// NewKVToSqlRowConverterForCols returns a KVToSqlConverter instance based on the list of rows passed in
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/rows/columns

func NewKVToSqlRowConverterForCols(cols []schema.Column) *KVToSqlRowConverter {
tagToSqlColIdx := make(map[uint64]int)
for i, col := range cols {
Expand All @@ -43,11 +50,21 @@ func NewKVToSqlRowConverterForCols(cols []schema.Column) *KVToSqlRowConverter {
}
}

// ConvertKVToSqlRow returns a sql.Row generated from the key and value provided.
func (conv *KVToSqlRowConverter) ConvertKVToSqlRow(k, v types.Value) (sql.Row, error) {
keyTup := k.(types.Tuple)
keyTup, ok := k.(types.Tuple)

if !ok {
return nil, errors.New("invalid key is not a tuple")
}

var valTup types.Tuple
if !types.IsNull(v) {
valTup = v.(types.Tuple)
valTup, ok = v.(types.Tuple)

if !ok {
return nil, errors.New("invalid value is not a tuple")
}
}

cols := make([]interface{}, conv.rowSize)
Expand Down Expand Up @@ -110,13 +127,18 @@ func (conv *KVToSqlRowConverter) processTuple(cols []interface{}, filled int, tu
return filled, nil
}

// KVGetFunc defines a function that returns a Key Value pair
type KVGetFunc func(ctx context.Context) (types.Value, types.Value, error)

// DoltMapIter uses a types.MapIterator to iterate over a types.Map and returns sql.Row instances that it reads and
// converts
type DoltMapIter struct {
kvGet KVGetFunc
conv *KVToSqlRowConverter
}

// NewDoltMapIterFromNomsMapItr returns an iterator which returns sql.Row instances read from a types.Map. The cols
// passed in are used to limit the values that are processed
func NewDoltMapIterFromNomsMapItr(mapItr types.MapIterator, cols []schema.Column) *DoltMapIter {
getFunc := func(ctx context.Context) (types.Value, types.Value, error) {
k, v, err := mapItr.Next(ctx)
Expand All @@ -133,13 +155,15 @@ func NewDoltMapIterFromNomsMapItr(mapItr types.MapIterator, cols []schema.Column
return NewDoltMapIter(getFunc, cols)
}

// NewDoltMapIter returns a new DoltMapIter
func NewDoltMapIter(keyValGet KVGetFunc, cols []schema.Column) *DoltMapIter {
return &DoltMapIter{
kvGet: keyValGet,
conv: NewKVToSqlRowConverterForCols(cols),
}
}

// Next returns the next sql.Row until all rows are returned at which point (nil, io.EOF) is returned.
func (dmi *DoltMapIter) Next(ctx context.Context) (sql.Row, error) {
k, v, err := dmi.kvGet(ctx)

Expand Down