Merge pull request #57346 from tallclair/psp-metrics-alt

Automatic merge from submit-queue. Temporary implementation of count metrics for PodSecurityPolicy **What this PR does / why we need it**: Alternative proposal to #57173 > We need rejection counts in order to turn on the PodSecurityPolicy controller. Comprehensive metrics were added for all admission controllers in 1.9, but backporting all those metrics was deemed to risky. So instead, this PR only enables the metrics on the PodSecurityPolicy controller. **Which issue(s) this PR fixes**: Fixes #55030 **Special notes for your reviewer**: Most of the diff is tests & boiler plate. Most prod code changes are contained in metrics.go, with a small hook in admission.go. This deviates from the metrics in HEAD, but some amount of drift between 1.8 and 1.9 is inevitable, due to the admission refactorings that went into 1.9. **Release note**: ```release-note Add prometheus metrics for the PodSecurityPolicy admission controller ```
kubernetes · Dec 20, 2017 · 24d0563 · 24d0563
2 parents 215590c + d0d6bf4
commit 24d0563
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 1 deletion.
diff --git a/plugin/pkg/admission/security/podsecuritypolicy/BUILD b/plugin/pkg/admission/security/podsecuritypolicy/BUILD
@@ -8,7 +8,10 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["admission.go"],
+    srcs = [
+        "admission.go",
+        "metrics.go",
+    ],
     deps = [
         "//pkg/api:go_default_library",
         "//pkg/apis/extensions:go_default_library",
@@ -20,6 +23,7 @@ go_library(
         "//pkg/security/podsecuritypolicy/util:go_default_library",
         "//pkg/serviceaccount:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
@@ -44,7 +48,10 @@ go_test(
         "//pkg/security/podsecuritypolicy:go_default_library",
         "//pkg/security/podsecuritypolicy/seccomp:go_default_library",
         "//pkg/security/podsecuritypolicy/util:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/prometheus/client_model/go:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

diff --git a/plugin/pkg/admission/security/podsecuritypolicy/admission.go b/plugin/pkg/admission/security/podsecuritypolicy/admission.go
@@ -111,6 +111,12 @@ func (a *podSecurityPolicyPlugin) SetInternalKubeInformerFactory(f informers.Sha
 //     with the validated PSP.  If we don't find any reject the pod and give all errors from the
 //     failed attempts.
 func (c *podSecurityPolicyPlugin) Admit(a admission.Attributes) error {
+	err := c.admit(a)
+	ObserveAdmit(err != nil, a)
+	return err
+}
+
+func (c *podSecurityPolicyPlugin) admit(a admission.Attributes) error {
 	if a.GetResource().GroupResource() != api.Resource("pods") {
 		return nil
 	}

diff --git a/plugin/pkg/admission/security/podsecuritypolicy/admission_test.go b/plugin/pkg/admission/security/podsecuritypolicy/admission_test.go
@@ -22,7 +22,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+	ptype "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -1651,6 +1654,7 @@ func testPSPAdmitAdvanced(testCaseName string, op kadmission.Operation, psps []*
 
 	originalPod := pod.DeepCopy()
 
+	admitCounter.Reset()
 	plugin := NewTestAdmission(informerFactory.Extensions().InternalVersion().PodSecurityPolicies().Lister())
 
 	attrs := kadmission.NewAttributesRecord(pod, oldPod, kapi.Kind("Pod").WithVersion("version"), "namespace", "", kapi.Resource("pods").WithVersion("version"), "", op, &user.DefaultInfo{})
@@ -1681,6 +1685,20 @@ func testPSPAdmitAdvanced(testCaseName string, op kadmission.Operation, psps []*
 	if !shouldPass && err == nil {
 		t.Errorf("%s: expected errors but received none", testCaseName)
 	}
+
+	wantLabels := map[string]string{
+		"operation":   string(op),
+		"group":       "",
+		"version":     "version",
+		"resource":    "pods",
+		"subresource": "",
+		"type":        "admit",
+		"rejected":    "false",
+	}
+	if !shouldPass {
+		wantLabels["rejected"] = "true"
+	}
+	expectMetric(t, wantLabels)
 }
 
 func TestAssignSecurityContext(t *testing.T) {
@@ -2104,3 +2122,39 @@ func groupIDPtr(i int) *int64 {
 	groupID := int64(i)
 	return &groupID
 }
+
+func expectMetric(t *testing.T, labelFilter map[string]string) {
+	metrics, err := prometheus.DefaultGatherer.Gather()
+	require.NoError(t, err)
+
+	counterSum := 0
+	for _, mf := range metrics {
+		if mf.GetName() != "apiserver_admission_controller_admission_latencies_seconds_count" {
+			continue // Ignore other metrics.
+		}
+		for _, metric := range mf.GetMetric() {
+			if !labelsMatch(metric, labelFilter) {
+				continue
+			}
+			counterSum += int(metric.GetCounter().GetValue())
+		}
+	}
+	assert.EqualValues(t, 1, counterSum,
+		"Wanted metric with labels %#+v", labelFilter)
+	if 1 != counterSum {
+		for _, mf := range metrics {
+			for _, metric := range mf.GetMetric() {
+				t.Logf("\tnear match: %s", metric.String())
+			}
+		}
+	}
+}
+
+func labelsMatch(metric *ptype.Metric, labelFilter map[string]string) bool {
+	for _, lp := range metric.GetLabel() {
+		if value, ok := labelFilter[lp.GetName()]; ok && lp.GetValue() != value {
+			return false
+		}
+	}
+	return true
+}
diff --git a/plugin/pkg/admission/security/podsecuritypolicy/metrics.go b/plugin/pkg/admission/security/podsecuritypolicy/metrics.go
@@ -0,0 +1,51 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podsecuritypolicy
+
+import (
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apiserver/pkg/admission"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "admission"
+)
+
+var (
+	admitCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "controller_admission_latencies_seconds_count",
+			Help:      "Admission controller counts, identified by name and broken out for each operation and API resource and type (validate or admit).",
+		},
+		[]string{"name", "type", "operation", "group", "version", "resource", "subresource", "rejected"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(admitCounter)
+}
+
+func ObserveAdmit(rejected bool, attr admission.Attributes) {
+	gvr := attr.GetResource()
+	labels := []string{PluginName, "admit", string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), strconv.FormatBool(rejected)}
+	admitCounter.WithLabelValues(labels...).Inc()
+}