Merge pull request prometheus#1644 from prometheus/beorn7/logging

Add missing logging of out-of-order samples
chowdhary987 · May 20, 2016 · f7ed2ff · f7ed2ff
2 parents dec5683 + 45e5775
commit f7ed2ff
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 3 deletions.
diff --git a/retrieval/scrape.go b/retrieval/scrape.go
@@ -496,6 +496,10 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, err error)
 		Value:     model.SampleValue(float64(duration) / float64(time.Second)),
 	}
 
-	sl.reportAppender.Append(healthSample)
-	sl.reportAppender.Append(durationSample)
+	if err := sl.reportAppender.Append(healthSample); err != nil {
+		log.With("sample", healthSample).With("error", err).Warn("Scrape health sample discarded")
+	}
+	if err := sl.reportAppender.Append(durationSample); err != nil {
+		log.With("sample", durationSample).With("error", err).Warn("Scrape duration sample discarded")
+	}
 }
diff --git a/rules/manager.go b/rules/manager.go
@@ -31,6 +31,7 @@ import (
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/template"
 	"github.com/prometheus/prometheus/util/strutil"
 )
@@ -270,8 +271,29 @@ func (g *Group) eval() {
 			if ar, ok := rule.(*AlertingRule); ok {
 				g.sendAlerts(ar, now)
 			}
+			var (
+				numOutOfOrder = 0
+				numDuplicates = 0
+			)
 			for _, s := range vector {
-				g.opts.SampleAppender.Append(s)
+				if err := g.opts.SampleAppender.Append(s); err != nil {
+					switch err {
+					case local.ErrOutOfOrderSample:
+						numOutOfOrder++
+						log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
+					case local.ErrDuplicateSampleForTimestamp:
+						numDuplicates++
+						log.With("sample", s).With("error", err).Debug("Rule evaluation result discarded")
+					default:
+						log.With("sample", s).With("error", err).Warn("Rule evaluation result discarded")
+					}
+				}
+			}
+			if numOutOfOrder > 0 {
+				log.With("numDropped", numOutOfOrder).Warn("Error on ingesting out-of-order result from rule evaluation")
+			}
+			if numDuplicates > 0 {
+				log.With("numDropped", numDuplicates).Warn("Error on ingesting results from rule evaluation with different value but same timestamp")
 			}
 		}(rule)
 	}