Add UUID to all metrics if requested

FerretDB · AlekSi · Oct 12, 2022 · Oct 10, 2022 · Oct 10, 2022 · Oct 11, 2022
commit 894b614d30c99c0ad25bed62c9edf508930dbbdd
@@ -28,6 +28,7 @@ import (
 
 	"github.com/alecthomas/kong"
 	"github.com/jackc/pgx/v4"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/tigrisdata/tigris-client-go/config"
 	"github.com/tigrisdata/tigris-client-go/driver"
 	"go.uber.org/zap"
@@ -259,7 +260,7 @@ func setupTigris(ctx context.Context, logger *zap.SugaredLogger) error {
 
 // run runs all setup commands.
 func run(ctx context.Context, logger *zap.SugaredLogger) error {
-	go debug.RunHandler(ctx, "127.0.0.1:8089", logger.Named("debug").Desugar())
+	go debug.RunHandler(ctx, "127.0.0.1:8089", prometheus.DefaultRegisterer, logger.Named("debug").Desugar())
 
 	if err := setupPostgres(ctx, logger); err != nil {
 		return err

@@ -42,15 +42,17 @@ import (
 var cli struct {
 	ListenAddr string `default:"127.0.0.1:27017"      help:"Listen address."`
 	ProxyAddr  string `default:"127.0.0.1:37017"      help:"Proxy address."`
-	DebugAddr  string `default:"127.0.0.1:8088"       help:"Debug address."`
+	DebugAddr  string `default:"127.0.0.1:8088"       help:"${help_debug_addr}"`
 	StateDir   string `default:"."                    help:"Process state directory."`
 	Mode       string `default:"${default_mode}"      help:"${help_mode}"             enum:"${enum_mode}"`
 
 	Log struct {
 		Level string `default:"${default_log_level}" help:"${help_log_level}"`
-		UUID  bool   `default:"false"                help:"Add instance UUID to log messages."`
+		UUID  bool   `default:"false"                help:"Add instance UUID to all log messages."`
 	} `embed:"" prefix:"log-"`
 
+	MetricsUUID bool `default:"false" help:"Add instance UUID to all metrics messages."`
+
 	Handler string `default:"pg" help:"${help_handler}"`
 
 	PostgresURL string `default:"postgres://postgres@127.0.0.1:5432/ferretdb" help:"PostgreSQL URL for 'pg' handler."`
@@ -80,6 +82,7 @@ var (
 			"default_log_level": zap.DebugLevel.String(),
 			"default_mode":      clientconn.AllModes[0],
 
+			"help_debug_addr": "Debug address for /debug/metrics, /debug/pprof, and similar HTTP handlers.",
 			"help_log_level": fmt.Sprintf(
 				"Log level: '%s'. Debug level also enables development mode.",
 				strings.Join(logLevels, "', '"),
@@ -107,24 +110,14 @@ func main() {
 	run()
 }
 
-// run sets up environment based on provided flags and runs FerretDB.
-func run() {
-	info := version.Get()
-
-	if cli.Version {
-		fmt.Fprintln(os.Stdout, "version:", info.Version)
-		fmt.Fprintln(os.Stdout, "commit:", info.Commit)
-		fmt.Fprintln(os.Stdout, "branch:", info.Branch)
-		fmt.Fprintln(os.Stdout, "dirty:", info.Dirty)
-		return
-	}
-
-	stateFile, err := filepath.Abs(filepath.Join(cli.StateDir, "state.json"))
+// setupState setups state provider.
+func setupState() (*state.Provider, string) {
+	f, err := filepath.Abs(filepath.Join(cli.StateDir, "state.json"))
 	if err != nil {
 		log.Fatalf("Failed to get path for state file: %s.", err)
 	}
 
-	p, err := state.NewProvider(stateFile)
+	p, err := state.NewProvider(f)
 	if err != nil {
 		log.Fatalf("Failed to create state provider: %s.", err)
 	}
@@ -134,12 +127,34 @@ func run() {
 		log.Fatalf("Failed to get state: %s.", err)
 	}
 
-	level, err := zapcore.ParseLevel(cli.Log.Level)
-	if err != nil {
-		log.Fatal(err)
+	return p, s.UUID
+}
+
+// setupMetrics setups Prometheus metrics registerer with some metrics.
+func setupMetrics(stateProvider *state.Provider, uuid string) prometheus.Registerer {
+	r := prometheus.WrapRegistererWith(
+		prometheus.Labels{"uuid": uuid},
+		prometheus.DefaultRegisterer,
+	)
+	m := stateProvider.MetricsCollector(false)
+
+	// Unless requested, don't add UUID to all metrics, but add it to one.
+	// See https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
+	if !cli.MetricsUUID {
+		r = prometheus.DefaultRegisterer
+		m = stateProvider.MetricsCollector(true)
 	}
 
-	logUUID := s.UUID
+	r.MustRegister(m)
+
+	return r
+}
+
+// setupLogger setups zap logger.
+func setupLogger(uuid string) *zap.Logger {
+	info := version.Get()
+
+	logUUID := uuid
 	startupFields := []zap.Field{
 		zap.String("version", info.Version),
 		zap.String("commit", info.Commit),
@@ -149,16 +164,43 @@ func run() {
 		zap.Reflect("buildEnvironment", info.BuildEnvironment.Map()),
 	}
 
-	// don't add UUID to all messages, but log it once at startup
+	// Similarly to Prometheus, unless requested, don't add UUID to all messages, but log it once at startup.
 	if !cli.Log.UUID {
 		logUUID = ""
-		startupFields = append(startupFields, zap.String("uuid", s.UUID))
+		startupFields = append(startupFields, zap.String("uuid", uuid))
+	}
+
+	level, err := zapcore.ParseLevel(cli.Log.Level)
+	if err != nil {
+		log.Fatal(err)
 	}
 
 	logging.Setup(level, logUUID)
-	logger := zap.L()
+	l := zap.L()
+
+	l.Info("Starting FerretDB "+info.Version+"...", startupFields...)
+
+	return l
+}
+
+// run sets up environment based on provided flags and runs FerretDB.
+func run() {
+	if cli.Version {
+		info := version.Get()
+
+		fmt.Fprintln(os.Stdout, "version:", info.Version)
+		fmt.Fprintln(os.Stdout, "commit:", info.Commit)
+		fmt.Fprintln(os.Stdout, "branch:", info.Branch)
+		fmt.Fprintln(os.Stdout, "dirty:", info.Dirty)
+
+		return
+	}
+
+	stateProvider, uuid := setupState()
+
+	metricsRegisterer := setupMetrics(stateProvider, uuid)
 
-	logger.Info("Starting FerretDB "+info.Version+"...", startupFields...)
+	logger := setupLogger(uuid)
 
 	ctx, stop := notifyAppTermination(context.Background())
 	go func() {
@@ -167,7 +209,7 @@ func run() {
 		stop()
 	}()
 
-	go debug.RunHandler(ctx, cli.DebugAddr, logger.Named("debug"))
+	go debug.RunHandler(ctx, cli.DebugAddr, metricsRegisterer, logger.Named("debug"))
 
 	h, err := registry.NewHandler(cli.Handler, &registry.NewHandlerOpts{
 		Ctx:    ctx,
@@ -195,7 +237,7 @@ func run() {
 		TestRecordsDir:  cli.Test.RecordsDir,
 	})
 
-	prometheus.DefaultRegisterer.MustRegister(l)
+	metricsRegisterer.MustRegister(l)
 
 	err = l.Run(ctx)
 	if err == nil || err == context.Canceled {

@@ -434,16 +434,6 @@ func (c *conn) handleOpMsg(ctx context.Context, msg *wire.OpMsg, cmd string) (*w
 	return nil, common.NewErrorMsg(common.ErrCommandNotFound, errMsg)
 }
 
-// Describe implements prometheus.Collector.
-func (c *conn) Describe(ch chan<- *prometheus.Desc) {
-	c.m.Describe(ch)
-}
-
-// Collect implements prometheus.Collector.
-func (c *conn) Collect(ch chan<- prometheus.Metric) {
-	c.m.Collect(ch)
-}
-
 // logResponse logs response's header and body and returns the log level that was used.
 //
 // The param `who` will be used in logs and should represent the type of the response,
@@ -475,3 +465,18 @@ func (c *conn) logResponse(who string, resHeader *wire.MsgHeader, resBody wire.M
 
 	return level
 }
+
+// Describe implements prometheus.Collector.
+func (c *conn) Describe(ch chan<- *prometheus.Desc) {
+	c.m.Describe(ch)
+}
+
+// Collect implements prometheus.Collector.
+func (c *conn) Collect(ch chan<- prometheus.Metric) {
+	c.m.Collect(ch)
+}
+
+// check interfaces
+var (
+	_ prometheus.Collector = (*conn)(nil)
+)
@@ -179,3 +179,8 @@ func (l *Listener) Describe(ch chan<- *prometheus.Desc) {
 func (l *Listener) Collect(ch chan<- prometheus.Metric) {
 	l.metrics.Collect(ch)
 }
+
+// check interfaces
+var (
+	_ prometheus.Collector = (*Listener)(nil)
+)
@@ -28,17 +28,17 @@ import (
 )
 
 // RunHandler runs debug handler.
-func RunHandler(ctx context.Context, addr string, l *zap.Logger) {
+func RunHandler(ctx context.Context, addr string, r prometheus.Registerer, l *zap.Logger) {
 	stdL, err := zap.NewStdLogAt(l, zap.WarnLevel)
 	if err != nil {
 		panic(err)
 	}
 
 	http.Handle("/debug/metrics", promhttp.InstrumentMetricHandler(
-		prometheus.DefaultRegisterer, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
+		r, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
 			ErrorLog:          stdL,
 			ErrorHandling:     promhttp.ContinueOnError,
-			Registry:          prometheus.DefaultRegisterer,
+			Registry:          r,
 			EnableOpenMetrics: true,
 		}),
 	))

@@ -0,0 +1,65 @@
+// Copyright 2021 FerretDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package state
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/FerretDB/FerretDB/internal/util/version"
+)
+
+// metricsCollector exposes provider's state as Prometheus metrics.
+type metricsCollector struct {
+	p               *Provider
+	addUUIDToMetric bool
+}
+
+// newMetricsCollector creates a new metricsCollector.
+//
+// If addUUIDToMetric is true, then the UUID is added to the Prometheus metric.
+func newMetricsCollector(p *Provider, addUUIDToMetric bool) *metricsCollector {
+	return &metricsCollector{
+		p:               p,
+		addUUIDToMetric: addUUIDToMetric,
+	}
+}
+
+// Describe implements prometheus.Collector.
+func (mc *metricsCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(mc, ch)
+}
+
+// Collect implements prometheus.Collector.
+func (mc *metricsCollector) Collect(ch chan<- prometheus.Metric) {
+	constLabels := prometheus.Labels{
+		"version": version.Get().Version,
+	}
+
+	if mc.addUUIDToMetric {
+		s, _ := mc.p.Get()
+		constLabels["uuid"] = s.UUID
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("ferretdb_up", "FerretDB instance state.", nil, constLabels),
+		prometheus.GaugeValue,
+		1,
+	)
+}
+
+// check interfaces
+var (
+	_ prometheus.Collector = (*metricsCollector)(nil)
+)
@@ -0,0 +1,80 @@
+// Copyright 2021 FerretDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package state
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/FerretDB/FerretDB/internal/util/version"
+)
+
+func TestMetrics(t *testing.T) {
+	t.Parallel()
+
+	filename := filepath.Join(t.TempDir(), "state.json")
+	p, err := NewProvider(filename)
+	require.NoError(t, err)
+
+	v := version.Get().Version
+
+	t.Run("WithUUID", func(t *testing.T) {
+		t.Parallel()
+
+		s, err := p.Get()
+		require.NoError(t, err)
+		uuid := s.UUID
+
+		mc := p.MetricsCollector(true)
+		problems, err := testutil.CollectAndLint(mc)
+		require.NoError(t, err)
+		require.Empty(t, problems)
+
+		expected := fmt.Sprintf(
+			`
+				# HELP ferretdb_up FerretDB instance state.
+				# TYPE ferretdb_up gauge
+				ferretdb_up{uuid=%q,version=%q} 1
+			`,
+			uuid, v,
+		)
+		assert.NoError(t, testutil.CollectAndCompare(mc, strings.NewReader(expected)))
+	})
+
+	t.Run("WithoutUUID", func(t *testing.T) {
+		t.Parallel()
+
+		mc := p.MetricsCollector(false)
+		problems, err := testutil.CollectAndLint(mc)
+		require.NoError(t, err)
+		require.Empty(t, problems)
+
+		expected := fmt.Sprintf(
+			`
+				# HELP ferretdb_up FerretDB instance state.
+				# TYPE ferretdb_up gauge
+				ferretdb_up{version=%q} 1
+			`,
+			v,
+		)
+		assert.NoError(t, testutil.CollectAndCompare(mc, strings.NewReader(expected)))
+	})
+}