feat(central-dashboard): Add Prometheus metrics with prom-client (#7639)

* Expose metrics with prom-client Expose default and custom metrics with prom-client [1]. Custom metrics: - rest_http_request_duration_seconds (Histogram) - response time - rest_http_request_total (Counter) - response count - app_info (Gauge) - app information: app name and version --- [1]: https://www.npmjs.com/package/prom-client Signed-off-by: Robert Gildein <robert.gildein@canonical.com> * add simple test and apply suggestions Signed-off-by: Robert Gildein <robert.gildein@canonical.com> * apply suggestions Signed-off-by: Robert Gildein <robert.gildein@canonical.com> --------- Signed-off-by: Robert Gildein <robert.gildein@canonical.com>
kubeflow · Sep 13, 2024 · f155d87 · f155d87
1 parent 01258d8
commit f155d87
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 7 deletions.
diff --git a/components/centraldashboard/app/metrics.ts b/components/centraldashboard/app/metrics.ts
@@ -0,0 +1,48 @@
+import express, {Request, Response} from "express";
+import client from "prom-client";
+import responseTime from "response-time";
+
+const appName = require("./../package.json").name;
+const appVersion = require("./../package.json").version;
+
+
+const restHttpRequestDuration = new client.Histogram({
+  name: "rest_http_request_duration_seconds",
+  help: "REST API response time in seconds",
+  labelNames: ["method", "path", "status"],
+});
+
+
+const restHttpRequestTotal = new client.Counter({
+  name: "rest_http_request_total",
+  help: "Total number of HTTP requests for REST API",
+  labelNames: ["method", "status"],
+});
+
+
+const appInfo = new client.Gauge({
+  name: "app_info",
+  help: "Information about application",
+  labelNames: ["version"],
+});
+
+
+export function enableMetricsCollection(app: express.Application) {
+  client.register.setDefaultLabels({ app: appName });  // set default labels
+  client.collectDefaultMetrics();
+
+  appInfo.labels({version: appVersion}).set(1);  // set value 1 for app_info with desired labels
+
+  app.get("/prometheus/metrics", async (req: Request, res: Response) => {
+    res.set("Content-Type", client.register.contentType);
+    return res.send(await client.register.metrics());
+  });
+
+  app.use(
+    responseTime((req: Request, res: Response, time: number) => {
+      restHttpRequestTotal.labels({ method: req.method, status: res.statusCode }).inc();
+      restHttpRequestDuration.labels(
+        { method: req.method, path: req.baseUrl, status: res.statusCode }).observe(time);
+    }),
+  );
+}
diff --git a/components/centraldashboard/app/metrics_test.ts b/components/centraldashboard/app/metrics_test.ts
@@ -0,0 +1,48 @@
+import express, {Request, Response} from "express";
+import {enableMetricsCollection} from "./metrics";
+import client from "prom-client";
+import {sendTestRequest} from './test_resources';
+
+
+describe("metrics", () => {
+    let app: express.Application;
+
+    beforeEach(() => {
+        jasmine.clock().install();
+        jasmine.clock().mockDate(new Date(1557705600000));
+        app = express();
+    });
+
+    describe("enableMetricsCollection", function () {
+        it("enable metrics", function() {
+            spyOn(client.register, "setDefaultLabels");
+            spyOn(client, "collectDefaultMetrics");
+            spyOn(app, "get");
+            spyOn(app, "use");
+
+            enableMetricsCollection(app);
+
+            expect(client.collectDefaultMetrics).toHaveBeenCalled();
+            expect(client.register.setDefaultLabels).toHaveBeenCalled();
+            expect(app.get).toHaveBeenCalledWith("/prometheus/metrics",  jasmine.any(Function));
+            expect(app.use).toHaveBeenCalledWith(jasmine.any(Function));
+        });
+
+        it("collect metrics", async () => {
+            const appName = require("./../package.json").name;
+            const appVersion = require("./../package.json").version;
+            const expectedAppInfo = `app_info{version="${appVersion}",app="${appName}"} 1`;
+            const addressInfo = app.listen(0).address();
+            const url = `http://localhost:${addressInfo.port}/prometheus/metrics`;
+
+            enableMetricsCollection(app);
+
+            const response = await sendTestRequest(url);
+            expect(response.includes(expectedAppInfo)).toBe(true);
+        });
+    });
+
+    afterEach(() => {
+        jasmine.clock().uninstall();
+    });
+});
diff --git a/components/centraldashboard/app/server.ts b/components/centraldashboard/app/server.ts
@@ -7,6 +7,7 @@ import {attachUser} from './attach_user_middleware';
 import {DefaultApi} from './clients/profile_controller';
 import {WorkgroupApi} from './api_workgroup';
 import {KubernetesService} from './k8s_service';
+import {enableMetricsCollection} from './metrics';
 import {getMetricsService} from './metrics_service_factory';
 import {PrometheusMetricsService} from "./prometheus_metrics_service";
 import {PrometheusDriver} from "prometheus-query";
@@ -33,6 +34,7 @@ const {
   REGISTRATION_FLOW = "true",
   PROMETHEUS_URL = undefined,
   METRICS_DASHBOARD = undefined,
+  COLLECT_METRICS = "true",
 } = process.env;
 
 
@@ -52,6 +54,13 @@ async function main() {
 
   console.info(`Using Profiles service at ${profilesServiceUrl}`);
   const profilesService = new DefaultApi(profilesServiceUrl);
+  const metrics: boolean = (COLLECT_METRICS.toLowerCase() === "true");
+
+  // Custom metrics configuration
+  if (metrics) {
+    console.info("Enabling the metrics collections to be accessible in the path `/prometheus/metrics`.");
+    enableMetricsCollection(app);
+  }
 
   app.use(express.json());
   app.use(express.static(frontEnd));

diff --git a/components/centraldashboard/manifests/base/deployment.yaml b/components/centraldashboard/manifests/base/deployment.yaml
@@ -46,4 +46,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: COLLECT_METRICS
+          value: CD_COLLECT_METRICS
       serviceAccountName: centraldashboard
diff --git a/components/centraldashboard/manifests/base/params.env b/components/centraldashboard/manifests/base/params.env
@@ -1,4 +1,5 @@
 CD_CLUSTER_DOMAIN=cluster.local
 CD_USERID_HEADER=kubeflow-userid
 CD_USERID_PREFIX=
-CD_REGISTRATION_FLOW=false
+CD_REGISTRATION_FLOW=false
+CD_COLLECT_METRICS=true
diff --git a/components/centraldashboard/package-lock.json b/components/centraldashboard/package-lock.json
diff --git a/components/centraldashboard/package.json b/components/centraldashboard/package.json
@@ -72,7 +72,9 @@
         "chartjs-plugin-crosshair": "^1.1.4",
         "express": "^4.19.2",
         "node-fetch": "^2.6.7",
+        "prom-client": "^15.1.3",
         "prometheus-query": "^3.3.2",
+        "response-time": "^2.3.2",
         "web-animations-js": "^2.3.2"
     },
     "devDependencies": {
@@ -87,6 +89,7 @@
         "@types/jasmine": "^3.4.2",
         "@types/node-fetch": "^2.5.2",
         "@types/puppeteer": "^2.0.0",
+        "@types/response-time": "^2.3.8",
         "@types/serve-static": "^1.15.5",
         "babel-loader": "^8.0.6",
         "clean-webpack-plugin": "^4.0.0",