Returning bytesRead as part of searchResult (blevesearch#1752)

* refactoring the bytes read while querying * bug fix: incrementing the bytes read at the right time * bug fix: fixing doc value and stored fields section bytes read computation * - updating stored fields' bytes read logic - bug fixes with respect to bytes read tracking in collector, tfr and docValueReader - fixing boolean searcher's BytesRead() API * - updating TestBytesRead and TestBytesReadStored with numbers after the bug fixes. * - updating the searcher interface and implementations with a SetBytesRead() API. Currently the only valid implmentations is with disjunct and conjunct searchers. - introducing BytesRead for newSnapshotIndexFieldDict. This is necessary for fuzzy, regex like queries. - updating the tests with the changes - updating rest of interface dependencies with no-op APIs in their implementations * - accounting the bytes read for prefix, range regexp queries in the seacher - minor code refactor with respect to tfr * a huge experimental change by introducing the concept of context right at the searcher level. this will essentially help in bytesRead tracking as well. * - using the IOstats reporter callback in term field reader and reporting the same as part of search result. - and yes, it works :D - removed the BytesRead and SetBytesRead API from searcher * - bug fixes with respect bytes read tracking in regexp queries - fixed a race condition around the newSnapshotIndexFieldDict - using the callbacks to return the bytes read value with respect to fieldDict in regexp, fuzzy etc type of queries, where there is a index.FieldDict creation involved (which loads the termDictionary for that field) - updated the tests reflecting the bug fixes. - removing the BytesRead API for index.TermFieldreader * - including the review comments and minor code cleanup * - addressing review comments, minor code cleanup - removing bytesRead from search.DocumentMatch * bug fixes with respect to reporting of the bytesRead * updating test plans * updating go.mod and go.sum * added some comments * minor renaming of function name
abhishekdas29 · Nov 11, 2022 · 1643dc3 · 1643dc3
1 parent 20e6eb5
commit 1643dc3
Show file tree

Hide file tree

Showing 90 changed files with 657 additions and 450 deletions.
diff --git a/document/document.go b/document/document.go
@@ -30,9 +30,14 @@ func init() {
 }
 
 type Document struct {
-	id              string  `json:"id"`
-	Fields          []Field `json:"fields"`
-	CompositeFields []*CompositeField
+	id               string  `json:"id"`
+	Fields           []Field `json:"fields"`
+	CompositeFields  []*CompositeField
+	StoredFieldsSize uint64
+}
+
+func (d *Document) StoredFieldsBytes() uint64 {
+	return d.StoredFieldsSize
 }
 
 func NewDocument(id string) *Document {

diff --git a/go.mod b/go.mod
@@ -5,23 +5,23 @@ go 1.18
 require (
 	github.com/RoaringBitmap/roaring v0.9.4
 	github.com/bits-and-blooms/bitset v1.2.0
-	github.com/blevesearch/bleve_index_api v1.0.4
+	github.com/blevesearch/bleve_index_api v1.0.5
 	github.com/blevesearch/geo v0.1.15
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/blevesearch/go-porterstemmer v1.0.3
 	github.com/blevesearch/goleveldb v1.0.1
 	github.com/blevesearch/gtreap v0.1.1
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.3
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.4
 	github.com/blevesearch/segment v0.9.0
 	github.com/blevesearch/snowball v0.6.1
 	github.com/blevesearch/snowballstem v0.9.0
 	github.com/blevesearch/upsidedown_store_api v1.0.1
 	github.com/blevesearch/vellum v1.0.9
-	github.com/blevesearch/zapx/v11 v11.3.6
-	github.com/blevesearch/zapx/v12 v12.3.6
-	github.com/blevesearch/zapx/v13 v13.3.6
-	github.com/blevesearch/zapx/v14 v14.3.6
-	github.com/blevesearch/zapx/v15 v15.3.6
+	github.com/blevesearch/zapx/v11 v11.3.7
+	github.com/blevesearch/zapx/v12 v12.3.7
+	github.com/blevesearch/zapx/v13 v13.3.7
+	github.com/blevesearch/zapx/v14 v14.3.7
+	github.com/blevesearch/zapx/v15 v15.3.7
 	github.com/couchbase/moss v0.2.0
 	github.com/golang/protobuf v1.3.2
 	github.com/spf13/cobra v1.4.0

diff --git a/go.sum b/go.sum
@@ -3,8 +3,8 @@ github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h
 github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
-github.com/blevesearch/bleve_index_api v1.0.4 h1:mtlzsyJjMIlDngqqB1mq8kPryUMIuEVVbRbJHOWEexU=
-github.com/blevesearch/bleve_index_api v1.0.4/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
+github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
+github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
 github.com/blevesearch/geo v0.1.15 h1:0NybEduqE5fduFRYiUKF0uqybAIFKXYjkBdXKYn7oA4=
 github.com/blevesearch/geo v0.1.15/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
@@ -18,8 +18,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
 github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.3 h1:2UzpR2dR5DvSZk8tVJkcQ7D5xhoK/UBelYw8ttBHrRQ=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.3/go.mod h1:eZrfp1y+lUh+DzFjUcTBUSnKGuunyFIpBIvqYVzJfvc=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0=
 github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
 github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
 github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -30,16 +30,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15a
 github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
 github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ=
 github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.6 h1:50jET4HUJ6eCqGxdhUt+mjybMvEX2MWyqLGtCx3yUgc=
-github.com/blevesearch/zapx/v11 v11.3.6/go.mod h1:B0CzJRj/pS7hJIroflRtFsa9mRHpMSucSgre0FVINns=
-github.com/blevesearch/zapx/v12 v12.3.6 h1:G304NHBLgQeZ+IHK/XRCM0nhHqAts8MEvHI6LhoDNM4=
-github.com/blevesearch/zapx/v12 v12.3.6/go.mod h1:iYi7tIKpauwU5os5wTxJITixr5Km21Hl365otMwdaP0=
-github.com/blevesearch/zapx/v13 v13.3.6 h1:vavltQHNdjQezhLZs5nIakf+w/uOa1oqZxB58Jy/3Ig=
-github.com/blevesearch/zapx/v13 v13.3.6/go.mod h1:X+FsTwCU8qOHtK0d/ArvbOH7qiIgViSQ1GQvcR6LSkI=
-github.com/blevesearch/zapx/v14 v14.3.6 h1:b9lub7TvcwUyJxK/cQtnN79abngKxsI7zMZnICU0WhE=
-github.com/blevesearch/zapx/v14 v14.3.6/go.mod h1:9X8W3XoikagU0rwcTqwZho7p9cC7m7zhPZO94S4wUvM=
-github.com/blevesearch/zapx/v15 v15.3.6 h1:VSswg/ysDxHgitcNkpUNtaTYS4j3uItpXWLAASphl6k=
-github.com/blevesearch/zapx/v15 v15.3.6/go.mod h1:5DbhhDTGtuQSns1tS2aJxJLPc91boXCvjOMeCLD1saM=
+github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw=
+github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA=
+github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI=
+github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0=
+github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8=
+github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s=
+github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20=
+github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w=
+github.com/blevesearch/zapx/v15 v15.3.7 h1:r8ZcNrlcMj2TmLlbNH16wZiL9reU0s7C2rAQKjFDtuE=
+github.com/blevesearch/zapx/v15 v15.3.7/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

diff --git a/index/scorch/builder_test.go b/index/scorch/builder_test.go
@@ -102,7 +102,7 @@ func checkIndex(t *testing.T, path string, term []byte, field string, expectCoun
 	}
 
 	// run a search for hello
-	tfr, err := r.TermFieldReader(term, field, false, false, false)
+	tfr, err := r.TermFieldReader(nil, term, field, false, false, false)
 	if err != nil {
 		t.Errorf("error accessing term field reader: %v", err)
 	} else {

diff --git a/index/scorch/reader_test.go b/index/scorch/reader_test.go
@@ -82,7 +82,7 @@ func TestIndexReader(t *testing.T) {
 	}()
 
 	// first look for a term that doesn't exist
-	reader, err := indexReader.TermFieldReader([]byte("nope"), "name", true, true, true)
+	reader, err := indexReader.TermFieldReader(nil, []byte("nope"), "name", true, true, true)
 	if err != nil {
 		t.Errorf("Error accessing term field reader: %v", err)
 	}
@@ -95,7 +95,7 @@ func TestIndexReader(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true)
+	reader, err = indexReader.TermFieldReader(nil, []byte("test"), "name", true, true, true)
 	if err != nil {
 		t.Errorf("Error accessing term field reader: %v", err)
 	}
@@ -145,17 +145,15 @@ func TestIndexReader(t *testing.T) {
 			},
 		},
 	}
-	tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc", true, true, true)
+	tfr, err := indexReader.TermFieldReader(nil, []byte("rice"), "desc", true, true, true)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
 	match, err = tfr.Next(nil)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
-	// Ignoring the BytesRead value, since it doesn't have
-	// relevance in this type of test
-	match.BytesRead = 0
+
 	if !reflect.DeepEqual(expectedMatch, match) {
 		t.Errorf("got %#v, expected %#v", match, expectedMatch)
 	}
@@ -165,7 +163,7 @@ func TestIndexReader(t *testing.T) {
 	}
 
 	// now test usage of advance
-	reader, err = indexReader.TermFieldReader([]byte("test"), "name", true, true, true)
+	reader, err = indexReader.TermFieldReader(nil, []byte("test"), "name", true, true, true)
 	if err != nil {
 		t.Errorf("Error accessing term field reader: %v", err)
 	}
@@ -196,7 +194,7 @@ func TestIndexReader(t *testing.T) {
 	}
 
 	// now test creating a reader for a field that doesn't exist
-	reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist", true, true, true)
+	reader, err = indexReader.TermFieldReader(nil, []byte("water"), "doesnotexist", true, true, true)
 	if err != nil {
 		t.Errorf("Error accessing term field reader: %v", err)
 	}

diff --git a/index/scorch/scorch_test.go b/index/scorch/scorch_test.go
@@ -1528,7 +1528,7 @@ func TestIndexTermReaderCompositeFields(t *testing.T) {
 		}
 	}()
 
-	termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all", true, true, true)
+	termFieldReader, err := indexReader.TermFieldReader(nil, []byte("mister"), "_all", true, true, true)
 	if err != nil {
 		t.Error(err)
 	}
@@ -2480,7 +2480,7 @@ func TestIndexSeekBackwardsStats(t *testing.T) {
 	}
 	defer reader.Close()
 
-	tfr, err := reader.TermFieldReader([]byte("cat"), "name", false, false, false)
+	tfr, err := reader.TermFieldReader(nil, []byte("cat"), "name", false, false, false)
 	if err != nil {
 		t.Fatalf("error getting term field readyer for name/cat: %v", err)
 	}

diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go
@@ -16,6 +16,7 @@ package scorch
 
 import (
 	"container/heap"
+	"context"
 	"encoding/binary"
 	"fmt"
 	"os"
@@ -37,6 +38,7 @@ import (
 // re usable, threadsafe levenshtein builders
 var lb1, lb2 *lev.LevenshteinAutomatonBuilder
 
+type diskStatsReporter segment.DiskStatsReporter
 type asynchSegmentResult struct {
 	dict    segment.TermDictionary
 	dictItr segment.DictionaryIterator
@@ -144,18 +146,17 @@ func (i *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 	randomLookup bool) (*IndexSnapshotFieldDict, error) {
 
 	results := make(chan *asynchSegmentResult)
+	var totalBytesRead uint64
 	for index, segment := range i.segment {
 		go func(index int, segment *SegmentSnapshot) {
-			var prevBytesRead uint64
-			prevBytesRead = segment.segment.BytesRead()
-
 			dict, err := segment.segment.Dictionary(field)
 			if err != nil {
 				results <- &asynchSegmentResult{err: err}
 			} else {
-				atomic.AddUint64(&i.parent.stats.TotBytesReadAtQueryTime,
-					segment.segment.BytesRead()-prevBytesRead)
-
+				if dictStats, ok := dict.(diskStatsReporter); ok {
+					atomic.AddUint64(&totalBytesRead,
+						dictStats.BytesRead())
+				}
 				if randomLookup {
 					results <- &asynchSegmentResult{dict: dict}
 				} else {
@@ -193,6 +194,7 @@ func (i *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 			}
 		}
 	}
+	rv.bytesRead = totalBytesRead
 	// after ensuring we've read all items on channel
 	if err != nil {
 		return nil, err
@@ -403,7 +405,7 @@ func (i *IndexSnapshot) DocCount() (uint64, error) {
 
 func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 	// FIXME could be done more efficiently directly, but reusing for simplicity
-	tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false)
+	tfr, err := i.TermFieldReader(nil, []byte(id), "_id", false, false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -430,13 +432,17 @@ func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 	segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum)
 
 	rvd := document.NewDocument(id)
-	prevBytesRead := i.segment[segmentIndex].segment.BytesRead()
 
 	err = i.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool {
 		if name == "_id" {
 			return true
 		}
 
+		// track uncompressed stored fields bytes as part of IO stats.
+		// However, ideally we'd need to track the compressed on-disk value
+		// Keeping that TODO for now until we have a cleaner way.
+		rvd.StoredFieldsSize += uint64(len(val))
+
 		// copy value, array positions to preserve them beyond the scope of this callback
 		value := append([]byte(nil), val...)
 		arrayPos := append([]uint64(nil), pos...)
@@ -464,9 +470,6 @@ func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 		return nil, err
 	}
 
-	if delta := i.segment[segmentIndex].segment.BytesRead() - prevBytesRead; delta > 0 {
-		atomic.AddUint64(&i.parent.stats.TotBytesReadAtQueryTime, delta)
-	}
 	return rvd, nil
 }
 
@@ -500,7 +503,7 @@ func (i *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
 
 func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) {
 	// FIXME could be done more efficiently directly, but reusing for simplicity
-	tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false)
+	tfr, err := i.TermFieldReader(nil, []byte(id), "_id", false, false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -518,10 +521,11 @@ func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err err
 	return next.ID, nil
 }
 
-func (is *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
+func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq,
 	includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
 	rv := is.allocTermFieldReaderDicts(field)
 
+	rv.ctx = ctx
 	rv.term = term
 	rv.field = field
 	rv.snapshot = is
@@ -541,13 +545,15 @@ func (is *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
 	if rv.dicts == nil {
 		rv.dicts = make([]segment.TermDictionary, len(is.segment))
 		for i, segment := range is.segment {
-			prevBytesRead := segment.segment.BytesRead()
+			segBytesRead := segment.segment.BytesRead()
+			rv.incrementBytesRead(segBytesRead)
 			dict, err := segment.segment.Dictionary(field)
 			if err != nil {
 				return nil, err
 			}
-			if bytesRead := segment.segment.BytesRead(); bytesRead > prevBytesRead {
-				atomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime, bytesRead-prevBytesRead)
+			if dictStats, ok := dict.(diskStatsReporter); ok {
+				bytesRead := dictStats.BytesRead()
+				rv.incrementBytesRead(bytesRead)
 			}
 			rv.dicts[i] = dict
 		}
@@ -571,13 +577,11 @@ func (is *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq,
 		rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i])
 
 		if bytesRead := rv.postings[i].BytesRead(); prevBytesReadPL < bytesRead {
-			atomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime,
-				bytesRead-prevBytesReadPL)
+			rv.incrementBytesRead(bytesRead - prevBytesReadPL)
 		}
 
 		if bytesRead := rv.iterators[i].BytesRead(); prevBytesReadItr < bytesRead {
-			atomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime,
-				bytesRead-prevBytesReadItr)
+			rv.incrementBytesRead(bytesRead - prevBytesReadItr)
 		}
 	}
 	atomic.AddUint64(&is.parent.stats.TotTermSearchersStarted, uint64(1))
@@ -634,6 +638,7 @@ func (i *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader
 		i.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{}
 	}
 	if uint64(len(i.fieldTFRs[tfr.field])) < i.getFieldTFRCacheThreshold() {
+		tfr.bytesRead = 0
 		i.fieldTFRs[tfr.field] = append(i.fieldTFRs[tfr.field], tfr)
 	}
 	i.m2.Unlock()
@@ -697,14 +702,10 @@ func (i *IndexSnapshot) documentVisitFieldTermsOnSegment(
 	}
 
 	if ssvOk && ssv != nil && len(vFields) > 0 {
-		prevBytesRead := ss.segment.BytesRead()
 		dvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs)
 		if err != nil {
 			return nil, nil, err
 		}
-		if delta := ss.segment.BytesRead() - prevBytesRead; delta > 0 {
-			atomic.AddUint64(&i.parent.stats.TotBytesReadAtQueryTime, delta)
-		}
 	}
 
 	if errCh != nil {
@@ -733,6 +734,13 @@ type DocValueReader struct {
 
 	currSegmentIndex int
 	currCachedFields []string
+
+	totalBytesRead uint64
+	bytesRead      uint64
+}
+
+func (dvr *DocValueReader) BytesRead() uint64 {
+	return dvr.totalBytesRead + dvr.bytesRead
 }
 
 func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,
@@ -750,11 +758,16 @@ func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,
 	if dvr.currSegmentIndex != segmentIndex {
 		dvr.currSegmentIndex = segmentIndex
 		dvr.currCachedFields = nil
+		dvr.totalBytesRead += dvr.bytesRead
+		dvr.bytesRead = 0
 	}
 
 	dvr.currCachedFields, dvr.dvs, err = dvr.i.documentVisitFieldTermsOnSegment(
 		dvr.currSegmentIndex, localDocNum, dvr.fields, dvr.currCachedFields, visitor, dvr.dvs)
 
+	if dvr.dvs != nil {
+		dvr.bytesRead = dvr.dvs.BytesRead()
+	}
 	return err
 }
 

diff --git a/index/scorch/snapshot_index_dict.go b/index/scorch/snapshot_index_dict.go
@@ -28,9 +28,14 @@ type segmentDictCursor struct {
 }
 
 type IndexSnapshotFieldDict struct {
-	snapshot *IndexSnapshot
-	cursors  []*segmentDictCursor
-	entry    index.DictEntry
+	snapshot  *IndexSnapshot
+	cursors   []*segmentDictCursor
+	entry     index.DictEntry
+	bytesRead uint64
+}
+
+func (i *IndexSnapshotFieldDict) BytesRead() uint64 {
+	return i.bytesRead
 }
 
 func (i *IndexSnapshotFieldDict) Len() int { return len(i.cursors) }