feat: add command_exit_code gauge

Signed-off-by: Utku Ozdemir <uoz@protonmail.com>
echoblag · Feb 8, 2022 · 2d21940 · 2d21940
1 parent 28c401e
commit 2d21940
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 27 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -46,10 +46,12 @@ jobs:
         with:
           # renovate: go
           go-version: 1.17.5
-      - name: Ensure gofmt
-        run: test -z "$(gofmt -s -d .)"
       - name: Ensure go.mod is already tidied
         run: go mod tidy && git diff --no-patch --exit-code
+      - name: Install gofumpt
+        run: go install mvdan.cc/gofumpt@v0.2.1
+      - name: Ensure code is properly formatted using gofumpt
+        run: test -z "$(gofumpt -s -d .)"
       - name: Run linters
         uses: golangci/golangci-lint-action@v2.5.2
         with:

diff --git a/cmd/nvidia_gpu_exporter/main.go b/cmd/nvidia_gpu_exporter/main.go
@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"os"
+
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -13,8 +16,6 @@ import (
 	webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag"
 	"github.com/utkuozdemir/nvidia_gpu_exporter/internal/exporter"
 	"gopkg.in/alecthomas/kingpin.v2"
-	"net/http"
-	"os"
 )
 
 const (

diff --git a/internal/exporter/csv_test.go b/internal/exporter/csv_test.go
@@ -1,8 +1,9 @@
 package exporter
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 const (

diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go
@@ -3,14 +3,15 @@ package exporter
 import (
 	"bytes"
 	"fmt"
-	"github.com/go-kit/log"
-	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
 	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // qField stands for query field - the field name before the query
@@ -48,6 +49,7 @@ type gpuExporter struct {
 	qFieldToMetricInfoMap map[qField]MetricInfo
 	nvidiaSmiCommand      string
 	failedScrapesTotal    prometheus.Counter
+	exitCode              prometheus.Gauge
 	gpuInfoDesc           *prometheus.Desc
 	logger                log.Logger
 }
@@ -59,7 +61,7 @@ func New(prefix string, nvidiaSmiCommand string, qFieldsRaw string, logger log.L
 	}
 
 	qFieldToMetricInfoMap := buildQFieldToMetricInfoMap(prefix, qFieldToRFieldMap)
-	//qFields := getKeys(qFieldToRFieldMap)
+	// qFields := getKeys(qFieldToRFieldMap)
 
 	infoLabels := getLabels(requiredFields)
 	e := gpuExporter{
@@ -73,6 +75,11 @@ func New(prefix string, nvidiaSmiCommand string, qFieldsRaw string, logger log.L
 			Name:      "failed_scrapes_total",
 			Help:      "Number of failed scrapes",
 		}),
+		exitCode: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: prefix,
+			Name:      "command_exit_code",
+			Help:      "Exit code of the last scrape command",
+		}),
 		gpuInfoDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(prefix, "", "gpu_info"),
 			fmt.Sprintf("A metric with a constant '1' value labeled by gpu %s.",
@@ -107,7 +114,7 @@ func buildQFieldToRFieldMap(logger log.Logger, qFieldsRaw string,
 		qFields = parsed
 	}
 
-	t, err := scrape(qFields, nvidiaSmiCommand)
+	_, t, err := scrape(qFields, nvidiaSmiCommand)
 	var rFields []rField
 	if err != nil {
 		_ = level.Warn(logger).Log("msg",
@@ -143,7 +150,9 @@ func (e *gpuExporter) Collect(ch chan<- prometheus.Metric) {
 	e.mutex.Lock()
 	defer e.mutex.Unlock()
 
-	t, err := scrape(e.qFields, e.nvidiaSmiCommand)
+	exitCode, t, err := scrape(e.qFields, e.nvidiaSmiCommand)
+	e.exitCode.Set(float64(exitCode))
+	ch <- e.exitCode
 	if err != nil {
 		_ = level.Error(e.logger).Log("error", err)
 		ch <- e.failedScrapesTotal
@@ -178,7 +187,7 @@ func (e *gpuExporter) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
-func scrape(qFields []qField, nvidiaSmiCommand string) (*table, error) {
+func scrape(qFields []qField, nvidiaSmiCommand string) (int, *table, error) {
 	qFieldsJoined := strings.Join(QFieldSliceToStringSlice(qFields), ",")
 
 	cmdAndArgs := strings.Fields(nvidiaSmiCommand)
@@ -193,15 +202,20 @@ func scrape(qFields []qField, nvidiaSmiCommand string) (*table, error) {
 
 	err := runCmd(cmd)
 	if err != nil {
-		return nil, fmt.Errorf("command failed. stderr: %s err: %w", stderr.String(), err)
+		exitCode := -1
+		if exitError, ok := err.(*exec.ExitError); ok {
+			exitCode = exitError.ExitCode()
+		}
+
+		return exitCode, nil, fmt.Errorf("command failed. stderr: %s err: %w", stderr.String(), err)
 	}
 
 	t, err := parseCSVIntoTable(strings.TrimSpace(stdout.String()), qFields)
 	if err != nil {
-		return nil, err
+		return -1, nil, err
 	}
 
-	return &t, nil
+	return 0, &t, nil
 }
 
 type MetricInfo struct {

diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go
@@ -2,22 +2,21 @@ package exporter
 
 import (
 	_ "embed"
-	"github.com/go-kit/log"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stretchr/testify/assert"
 	"os/exec"
 	"strings"
 	"testing"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
 )
 
 const (
 	delta = 1e-9
 )
 
-var (
-	//go:embed _query-test.txt
-	queryTest string
-)
+//go:embed _query-test.txt
+var queryTest string
 
 func assertFloat(t *testing.T, expected, actual float64) bool {
 	return assert.InDelta(t, expected, actual, delta)
@@ -208,8 +207,9 @@ end:
 
 	metricsJoined := strings.Join(metrics, "\n")
 
-	assert.Len(t, metrics, 9)
+	assert.Len(t, metrics, 10)
 	assert.Contains(t, metricsJoined, "aaa_gpu_info")
+	assert.Contains(t, metricsJoined, "command_exit_code")
 	assert.Contains(t, metricsJoined, "aaa_name")
 	assert.Contains(t, metricsJoined, "aaa_fan_speed_ratio")
 	assert.Contains(t, metricsJoined, "aaa_memory_used_bytes")
@@ -240,6 +240,9 @@ end:
 		}
 	}
 
-	assert.Len(t, metrics, 1)
-	assert.Contains(t, metrics[0], "aaa_failed_scrapes_total")
+	assert.Len(t, metrics, 2)
+	metricsJoined := strings.Join(metrics, "\n")
+
+	assert.Contains(t, metricsJoined, "aaa_failed_scrapes_total")
+	assert.Contains(t, metricsJoined, "aaa_command_exit_code")
 }
diff --git a/internal/exporter/fields_test.go b/internal/exporter/fields_test.go
@@ -2,9 +2,10 @@ package exporter
 
 import (
 	_ "embed"
-	"github.com/stretchr/testify/assert"
 	"os/exec"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 var (

diff --git a/internal/exporter/util_test.go b/internal/exporter/util_test.go
@@ -2,11 +2,12 @@ package exporter
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"math"
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestToSnakeCase(t *testing.T) {