From 34f0af30cff30c7319d474f8b034c685931e8b51 Mon Sep 17 00:00:00 2001 From: Fabian Reinartz Date: Thu, 21 Sep 2017 15:27:21 +0200 Subject: [PATCH] vendor: update prometheus/tsdb --- vendor/github.com/prometheus/tsdb/compact.go | 2 +- vendor/github.com/prometheus/tsdb/head.go | 20 ++++++++----- vendor/github.com/prometheus/tsdb/postings.go | 29 +++++++++++++++++-- vendor/github.com/prometheus/tsdb/wal.go | 15 ++++------ vendor/vendor.json | 14 ++++----- 5 files changed, 52 insertions(+), 28 deletions(-) diff --git a/vendor/github.com/prometheus/tsdb/compact.go b/vendor/github.com/prometheus/tsdb/compact.go index a3bc7d17a38..17a4da04181 100644 --- a/vendor/github.com/prometheus/tsdb/compact.go +++ b/vendor/github.com/prometheus/tsdb/compact.go @@ -457,7 +457,7 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta, indexr := b.Index() - all, err := indexr.Postings("", "") + all, err := indexr.Postings(allPostingsKey.Name, allPostingsKey.Value) if err != nil { return err } diff --git a/vendor/github.com/prometheus/tsdb/head.go b/vendor/github.com/prometheus/tsdb/head.go index a5ce94e4557..72973c46c89 100644 --- a/vendor/github.com/prometheus/tsdb/head.go +++ b/vendor/github.com/prometheus/tsdb/head.go @@ -190,6 +190,10 @@ func (h *Head) ReadWAL() error { r := h.wal.Reader() mint := h.MinTime() + // Track number of samples that referenced a series we don't know about + // for error reporting. + var unknownRefs int + seriesFunc := func(series []RefSeries) error { for _, s := range series { h.getOrCreateWithID(s.Ref, s.Labels.Hash(), s.Labels) @@ -207,7 +211,7 @@ func (h *Head) ReadWAL() error { } ms := h.series.getByID(s.Ref) if ms == nil { - h.logger.Log("msg", "unknown series reference in WAL", "ref", s.Ref) + unknownRefs++ continue } _, chunkCreated := ms.append(s.T, s.V) @@ -230,6 +234,10 @@ func (h *Head) ReadWAL() error { return nil } + if unknownRefs > 0 { + h.logger.Log("msg", "unknown series references in WAL samples", "count", unknownRefs) + } + if err := r.Read(seriesFunc, samplesFunc, deletesFunc); err != nil { return errors.Wrap(err, "consume WAL") } @@ -267,12 +275,10 @@ func (h *Head) Truncate(mint int64) error { start = time.Now() - p, err := h.indexRange(mint, math.MaxInt64).Postings("", "") - if err != nil { - return err + keep := func(id uint64) bool { + return h.series.getByID(id) != nil } - - if err := h.wal.Truncate(mint, p); err == nil { + if err := h.wal.Truncate(mint, keep); err == nil { h.logger.Log("msg", "WAL truncation completed", "duration", time.Since(start)) } else { h.logger.Log("msg", "WAL truncation failed", "err", err, "duration", time.Since(start)) @@ -1038,8 +1044,6 @@ func (s *stripeSeries) getOrSet(hash uint64, series *memSeries) (*memSeries, boo return prev, false } s.hashes[i].set(hash, series) - - s.hashes[i][hash] = append(s.hashes[i][hash], series) s.locks[i].Unlock() i = series.ref & stripeMask diff --git a/vendor/github.com/prometheus/tsdb/postings.go b/vendor/github.com/prometheus/tsdb/postings.go index 97a29ab197a..0e51b221b34 100644 --- a/vendor/github.com/prometheus/tsdb/postings.go +++ b/vendor/github.com/prometheus/tsdb/postings.go @@ -45,7 +45,7 @@ func (p *memPostings) get(name, value string) Postings { return newListPostings(l) } -var allLabel = labels.Label{} +var allPostingsKey = labels.Label{} // add adds a document to the index. The caller has to ensure that no // term argument appears twice. @@ -53,13 +53,36 @@ func (p *memPostings) add(id uint64, lset labels.Labels) { p.mtx.Lock() for _, l := range lset { - p.m[l] = append(p.m[l], id) + p.addFor(id, l) } - p.m[allLabel] = append(p.m[allLabel], id) + p.addFor(id, allPostingsKey) p.mtx.Unlock() } +func (p *memPostings) addFor(id uint64, l labels.Label) { + list := append(p.m[l], id) + p.m[l] = list + + // There is no guarantee that no higher ID was inserted before as they may + // be generated independently before adding them to postings. + // We repair order violations on insert. The invariant is that the first n-1 + // items in the list are already sorted. + for i := len(list) - 1; i >= 1; i-- { + if list[i] >= list[i-1] { + break + } + list[i], list[i-1] = list[i-1], list[i] + } +} + +func expandPostings(p Postings) (res []uint64, err error) { + for p.Next() { + res = append(res, p.At()) + } + return res, p.Err() +} + // Postings provides iterative access over a postings list. type Postings interface { // Next advances the iterator and returns true if another value was found. diff --git a/vendor/github.com/prometheus/tsdb/wal.go b/vendor/github.com/prometheus/tsdb/wal.go index 27984ea0ce7..c52bca86d64 100644 --- a/vendor/github.com/prometheus/tsdb/wal.go +++ b/vendor/github.com/prometheus/tsdb/wal.go @@ -71,7 +71,7 @@ type WAL interface { LogSeries([]RefSeries) error LogSamples([]RefSample) error LogDeletes([]Stone) error - Truncate(int64, Postings) error + Truncate(mint int64, keep func(uint64) bool) error Close() error } @@ -87,7 +87,7 @@ func (w nopWAL) Reader() WALReader { return w } func (nopWAL) LogSeries([]RefSeries) error { return nil } func (nopWAL) LogSamples([]RefSample) error { return nil } func (nopWAL) LogDeletes([]Stone) error { return nil } -func (nopWAL) Truncate(int64, Postings) error { return nil } +func (nopWAL) Truncate(int64, func(uint64) bool) error { return nil } func (nopWAL) Close() error { return nil } // WALReader reads entries from a WAL. @@ -272,8 +272,9 @@ func (w *SegmentWAL) putBuffer(b *encbuf) { w.buffers.Put(b) } -// Truncate deletes the values prior to mint and the series entries not in p. -func (w *SegmentWAL) Truncate(mint int64, p Postings) error { +// Truncate deletes the values prior to mint and the series which the keep function +// does not indiciate to preserve. +func (w *SegmentWAL) Truncate(mint int64, keep func(uint64) bool) error { // The last segment is always active. if len(w.files) < 2 { return nil @@ -314,7 +315,6 @@ func (w *SegmentWAL) Truncate(mint int64, p Postings) error { activeSeries = []RefSeries{} ) -Loop: for r.next() { rt, flag, byt := r.at() @@ -328,10 +328,7 @@ Loop: activeSeries = activeSeries[:0] for _, s := range series { - if !p.Seek(s.Ref) { - break Loop - } - if p.At() == s.Ref { + if keep(s.Ref) { activeSeries = append(activeSeries, s) } } diff --git a/vendor/vendor.json b/vendor/vendor.json index 9529d670020..2ab1a537a7c 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -871,22 +871,22 @@ "revisionTime": "2016-04-11T19:08:41Z" }, { - "checksumSHA1": "B5ndMoK8lqgFJ8xUZ/0V4zCpUw0=", + "checksumSHA1": "evkeOdR0mTFS7yyREas6oa1QvHY=", "path": "github.com/prometheus/tsdb", - "revision": "f39388c9af818643192c4acb64ed90e56628b5a7", - "revisionTime": "2017-09-19T13:41:53Z" + "revision": "69f105f4f9478e929ef2a7d7553a7558b1de5c84", + "revisionTime": "2017-09-21T12:57:51Z" }, { "checksumSHA1": "Gua979gmISm4cJP/fR2hL8m5To8=", "path": "github.com/prometheus/tsdb/chunks", - "revision": "f39388c9af818643192c4acb64ed90e56628b5a7", - "revisionTime": "2017-09-19T13:41:53Z" + "revision": "69f105f4f9478e929ef2a7d7553a7558b1de5c84", + "revisionTime": "2017-09-21T12:57:51Z" }, { "checksumSHA1": "zhmlvc322RH1L3l9DaA9d/HVVWs=", "path": "github.com/prometheus/tsdb/labels", - "revision": "f39388c9af818643192c4acb64ed90e56628b5a7", - "revisionTime": "2017-09-19T13:41:53Z" + "revision": "69f105f4f9478e929ef2a7d7553a7558b1de5c84", + "revisionTime": "2017-09-21T12:57:51Z" }, { "checksumSHA1": "5SYLEhADhdBVZAGPVHWggQl7H8k=",