diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..888ffaa --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vscode +.envrc diff --git a/README.md b/README.md index 7182109..1d635e6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,63 @@ # starter-k8s -A starter project to learn and test Kubernetes concepts + +This repo has files to quickstart a Kubernetes project or experiment with Kubernetes + +## Usage + +```bash +# Create a cluster in minikube, start it, and deploy several platform charts on it +make + +# Create a cluster in minikube +make start + +# Create a cluster in EKS +make start CLUSTER_TYPE="eks" + +# Deploy Prometheus, Grafana, Alertmanager, Gatekeeper (and custom templates & constraints), +# Falco, Robusta, Vault, cert-manager +make helmfile_sync + +# Delete minikube cluster +make delete + +# Delete eks cluster +make delete CLUSTER_TYPE="eks" + +# Follow Kubernetes audit log +make audit_log + +# Open proxy connection to Grafana +make proxy_grafana + +# Install microservices demo +make install_demo +``` + +Check more targets at `makefile`. + +## Directories + + + clusters: files to create and configure different cluster types + + charts: files to deploy and configure several Helm charts + + exercises: files to run commands to practice different Kubernetes concepts + +## Configuration + +### Alertmanager (Opsgenie, Mailtrap) + +**Alertmanager** takes Prometheus rules firing and sends an alert to a receiver application. + +To use **Opsgenie** with Alertmanager, copy `sample.envrc` to `.envrc`, and set your API key and team id in that file. Then load its values into environment before deploying charts, with `source .envrc` or using [direnv](https://direnv.net/). + +To use **Mailtrap** as an alternative, edit `./charts/prometheus/am-mailtrap.yaml` with your user and password, and edit `./charts/helmfile-observability`, switch commenting these lines so they look like this: +``` + - ./prometheus/am-mailtrap.yaml + # - ./prometheus/am-opsgenie.yaml +``` + +If you don't want to configure any alert receiver, comment both lines, and everything under the `set:` directive of `promstack` chart. + +### Robusta + +Generate your Robusta configuration and set relevant values on `.envrc` as previously, or comment out the whole robusta chart block on `./charts/helmfile-observability.yaml` diff --git a/charts/falco/falco_rule_gatekeeper_ac.yaml b/charts/falco/falco_rule_gatekeeper_ac.yaml new file mode 100644 index 0000000..a682d7a --- /dev/null +++ b/charts/falco/falco_rule_gatekeeper_ac.yaml @@ -0,0 +1,11 @@ +- rule: Admission Controller Block + desc: > + Detected an admission controller blocking a deployment + condition: kevt and jevt.value[/responseStatus/status]="Failure" and jevt.value[/responseStatus/reason]="Forbidden" + output: > + Admission controller blocked deployment + (user=%ka.user.name verb=%ka.verb message=jevt.value[/responseObject/message] sourceIps=jevt.value[/sourceIPs]) + priority: WARNING + source: k8s_audit + tags: [k8s] + diff --git a/charts/falco/values.yaml b/charts/falco/values.yaml new file mode 100644 index 0000000..13dd396 --- /dev/null +++ b/charts/falco/values.yaml @@ -0,0 +1,9 @@ +auditLog: + enabled: true +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + memory: 256Mi + cpu: 50m diff --git a/charts/gatekeeper/chart-constraints/.helmignore b/charts/gatekeeper/chart-constraints/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/gatekeeper/chart-constraints/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/gatekeeper/chart-constraints/Chart.yaml b/charts/gatekeeper/chart-constraints/Chart.yaml new file mode 100644 index 0000000..48d7d0e --- /dev/null +++ b/charts/gatekeeper/chart-constraints/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: chart-constraints +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +# dependencies: +# - name: chart-constraint-templates +# version: "0.1.0" +# repository: "file://../chart-constraint-templates" \ No newline at end of file diff --git a/charts/gatekeeper/chart-constraints/templates/NOTES.txt b/charts/gatekeeper/chart-constraints/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/gatekeeper/chart-constraints/templates/_helpers.tpl b/charts/gatekeeper/chart-constraints/templates/_helpers.tpl new file mode 100644 index 0000000..7f9a73a --- /dev/null +++ b/charts/gatekeeper/chart-constraints/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "chart-policies.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart-policies.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart-policies.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart-policies.labels" -}} +helm.sh/chart: {{ include "chart-policies.chart" . }} +{{ include "chart-policies.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart-policies.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart-policies.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart-policies.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart-policies.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/gatekeeper/chart-constraints/templates/c-label.yaml b/charts/gatekeeper/chart-constraints/templates/c-label.yaml new file mode 100644 index 0000000..6d88ecf --- /dev/null +++ b/charts/gatekeeper/chart-constraints/templates/c-label.yaml @@ -0,0 +1,15 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredLabels +metadata: + name: ns-must-have-gk +spec: + match: + kinds: + - apiGroups: ["apps"] + kinds: ["Deployment"] + scope: "Namespaced" + namespaces: + - "tennant" + parameters: + labels: + - {{ .Values.constraint.label }} diff --git a/charts/gatekeeper/chart-constraints/templates/c-privileged-container.yaml b/charts/gatekeeper/chart-constraints/templates/c-privileged-container.yaml new file mode 100644 index 0000000..d4c1dea --- /dev/null +++ b/charts/gatekeeper/chart-constraints/templates/c-privileged-container.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: PrivilegedContainers +metadata: + name: privileged-container +spec: + enforcementAction: deny + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + namespaces: + - "tennant" + diff --git a/charts/gatekeeper/chart-constraints/templates/tests/test-connection.yaml b/charts/gatekeeper/chart-constraints/templates/tests/test-connection.yaml new file mode 100644 index 0000000..0ead54c --- /dev/null +++ b/charts/gatekeeper/chart-constraints/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "chart-policies.fullname" . }}-test-connection" + labels: + {{- include "chart-policies.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "chart-policies.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/gatekeeper/chart-constraints/values.yaml b/charts/gatekeeper/chart-constraints/values.yaml new file mode 100644 index 0000000..ebeb550 --- /dev/null +++ b/charts/gatekeeper/chart-constraints/values.yaml @@ -0,0 +1,79 @@ +# Default values for chart-policies. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: false + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +#------------------------------------------------------------------------------------------------------------------ + +constraint: + label: gatekeeper diff --git a/charts/gatekeeper/chart-templates/.helmignore b/charts/gatekeeper/chart-templates/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/gatekeeper/chart-templates/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/gatekeeper/chart-templates/Chart.yaml b/charts/gatekeeper/chart-templates/Chart.yaml new file mode 100644 index 0000000..8cce94c --- /dev/null +++ b/charts/gatekeeper/chart-templates/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: chart-constraint-templates +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/charts/gatekeeper/chart-templates/templates/NOTES.txt b/charts/gatekeeper/chart-templates/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/gatekeeper/chart-templates/templates/_helpers.tpl b/charts/gatekeeper/chart-templates/templates/_helpers.tpl new file mode 100644 index 0000000..7f9a73a --- /dev/null +++ b/charts/gatekeeper/chart-templates/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "chart-policies.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart-policies.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart-policies.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart-policies.labels" -}} +helm.sh/chart: {{ include "chart-policies.chart" . }} +{{ include "chart-policies.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart-policies.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart-policies.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart-policies.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart-policies.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/gatekeeper/chart-templates/templates/t-label.yaml b/charts/gatekeeper/chart-templates/templates/t-label.yaml new file mode 100644 index 0000000..2a1db99 --- /dev/null +++ b/charts/gatekeeper/chart-templates/templates/t-label.yaml @@ -0,0 +1,31 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: k8srequiredlabels +spec: + crd: + spec: + names: + kind: K8sRequiredLabels + validation: + # Schema for the `parameters` field + openAPIV3Schema: + type: object + properties: + labels: + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package k8srequiredlabels + + violation[{"msg": msg, "details": {"missing_labels": missing}}] { + provided := {label | input.review.object.metadata.labels[label]} + required := {label | label := input.parameters.labels[_]} + missing := required - provided + count(missing) > {{ .Values.constraint.test }} + msg := sprintf("you must provide labels: %v", [missing]) + } + diff --git a/charts/gatekeeper/chart-templates/templates/t-privileged-container.yaml b/charts/gatekeeper/chart-templates/templates/t-privileged-container.yaml new file mode 100644 index 0000000..51c0a6f --- /dev/null +++ b/charts/gatekeeper/chart-templates/templates/t-privileged-container.yaml @@ -0,0 +1,25 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: privilegedcontainers +spec: + crd: + spec: + names: + kind: PrivilegedContainers + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package privilegedcontainers + + violation[{"msg": msg, "details": {}}] { + c := input_containers[_] + c.securityContext.privileged + msg := sprintf("Privileged container is not allowed: %v", [c.name]) + } + input_containers[c] { + c := input.review.object.spec.containers[_] + } + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } diff --git a/charts/gatekeeper/chart-templates/templates/tests/test-connection.yaml b/charts/gatekeeper/chart-templates/templates/tests/test-connection.yaml new file mode 100644 index 0000000..0ead54c --- /dev/null +++ b/charts/gatekeeper/chart-templates/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "chart-policies.fullname" . }}-test-connection" + labels: + {{- include "chart-policies.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "chart-policies.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/gatekeeper/chart-templates/values.yaml b/charts/gatekeeper/chart-templates/values.yaml new file mode 100644 index 0000000..6f4ed7b --- /dev/null +++ b/charts/gatekeeper/chart-templates/values.yaml @@ -0,0 +1,80 @@ +# Default values for chart-policies. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: false + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +# -------------------------------------------------------------------------------------------- + +constraint: + test: 0 + diff --git a/charts/helmfile-falco.yaml b/charts/helmfile-falco.yaml new file mode 100644 index 0000000..99bd12f --- /dev/null +++ b/charts/helmfile-falco.yaml @@ -0,0 +1,16 @@ +--- +repositories: + - name: falcosecurity + url: https://falcosecurity.github.io/charts + +releases: + + - name: falco + chart: falcosecurity/falco + namespace: falco + createNamespace: true + installed: true + wait: true + waitForJobs: true + values: + - ./falco/values.yaml diff --git a/charts/helmfile-gatekeeper.yaml b/charts/helmfile-gatekeeper.yaml new file mode 100644 index 0000000..bb49717 --- /dev/null +++ b/charts/helmfile-gatekeeper.yaml @@ -0,0 +1,40 @@ +--- +repositories: + - name: gatekeeper + url: https://open-policy-agent.github.io/gatekeeper/charts + +releases: + + - name: gatekeeper-system + chart: gatekeeper/gatekeeper + namespace: gatekeeper + version: "3.8.1" + createNamespace: true + installed: true + wait: true + waitForJobs: true + + - name: gatekeeper-templates + chart: ./gatekeeper/chart-templates/ + namespace: gatekeeper + version: "0.1.0" + installed: true + wait: true + waitForJobs: true + values: + - gatekeeper/chart-templates/values.yaml + needs: + - gatekeeper/gatekeeper-system + + - name: gatekeeper-constraints + chart: ./gatekeeper/chart-constraints/ + namespace: gatekeeper + version: "0.1.0" + installed: true + wait: true + waitForJobs: true + values: + - gatekeeper/chart-constraints/values.yaml + needs: + - gatekeeper/gatekeeper-system + - gatekeeper/gatekeeper-templates diff --git a/charts/helmfile-observability.yaml b/charts/helmfile-observability.yaml new file mode 100644 index 0000000..9f88954 --- /dev/null +++ b/charts/helmfile-observability.yaml @@ -0,0 +1,64 @@ +--- +repositories: + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts + - name: sstarcher + url: https://shanestarcher.com/helm-charts/ + - name : robusta + url: https://robusta-charts.storage.googleapis.com + +releases: + + - name: promstack + chart: prometheus-community/kube-prometheus-stack + namespace: promstack + version: "35.0.3" + createNamespace: true + installed: true + wait: true + values: + - ./prometheus/am-general.yaml + # - ./prometheus/am-mailtrap.yaml + - ./prometheus/am-opsgenie.yaml + set: + - name: alertmanager.config.global.opsgenie_api_key + value: {{ requiredEnv "opsgenie_api_key" }} + - name: alertmanager.config.receivers[0].opsgenie_configs[0].responders[0].id + value: {{ requiredEnv "opsgenie_responder_id" }} + + - name: prometheus-extras + chart: ./prometheus/chart-prometheus-extras + namespace: promstack + version: "35.0.3" + createNamespace: true + installed: true + needs: + - promstack/promstack + + - name: helm-exporter + chart: sstarcher/helm-exporter + namespace: helm-exporter + version: "1.2.2+6766a95" + createNamespace: true + installed: true + + - name: robusta + chart: robusta/robusta + # version: "" + namespace: robusta + createNamespace: true + installed: true + values: + - ./robusta/am-robusta.yaml + - ./robusta/generated_values.yaml + set: + - name: globalConfig.signing_key + value: {{ requiredEnv "robusta_signing_key" | quote }} + - name: globalConfig.account_id + value: {{ requiredEnv "robusta_account_id" | quote }} + - name: sinksConfig[0].token + value: {{ requiredEnv "robusta_sink_ui" | quote }} + - name: rsa.prv + value: {{ requiredEnv "robusta_private_key" | quote }} + - name: rsa.pub + value: {{ requiredEnv "robusta_public_key" | quote }} diff --git a/charts/helmfile-secrets.yaml b/charts/helmfile-secrets.yaml new file mode 100644 index 0000000..e25f100 --- /dev/null +++ b/charts/helmfile-secrets.yaml @@ -0,0 +1,27 @@ +--- +repositories: + - name: hashicorp + url: https://helm.releases.hashicorp.com + - name: jetstack + url: https://charts.jetstack.io + +releases: + + - name: vault + chart: hashicorp/vault + namespace: vault + version: "0.19.0" + createNamespace: true + installed: true + values: + - ./vault/values.yaml + + - name: cert-manager + chart: jetstack/cert-manager + namespace: cert-manager + version: "1.8.2" + createNamespace: true + installed: true + values: + - installCRDs: true + diff --git a/charts/helmfile.yaml b/charts/helmfile.yaml new file mode 100644 index 0000000..63778bf --- /dev/null +++ b/charts/helmfile.yaml @@ -0,0 +1,8 @@ +--- +context: starter-k8s + +helmfiles: +- ./helmfile-gatekeeper.yaml +- ./helmfile-secrets.yaml +- ./helmfile-observability.yaml +- ./helmfile-falco.yaml \ No newline at end of file diff --git a/charts/prometheus/am-general.yaml b/charts/prometheus/am-general.yaml new file mode 100644 index 0000000..008f730 --- /dev/null +++ b/charts/prometheus/am-general.yaml @@ -0,0 +1,19 @@ +grafana: + defaultDashboardsEnabled: 0 +alertmanager: + config: + global: + resolve_timeout: 5m + route: + group_wait: 20s + group_interval: 4m + repeat_interval: 4h + routes: + - match: + alertname: NodeClockNotSynchronising + receiver: "null" + - match: + alertname: etcdInsufficientMembers + receiver: "null" + receivers: + - name: "null" diff --git a/charts/prometheus/am-mailtrap.sample.yaml b/charts/prometheus/am-mailtrap.sample.yaml new file mode 100644 index 0000000..fbc569a --- /dev/null +++ b/charts/prometheus/am-mailtrap.sample.yaml @@ -0,0 +1,12 @@ +alertmanager: + config: + route: + receiver: 'email-k8s-admin' + receivers: + - name: 'email-k8s-admin' + email_configs: + - to: k8s-admin@example.com + from: email-k8s-admin@alertmanager.com + smarthost: smtp.mailtrap.io:587 + auth_username: replace_username + auth_password: replace_password \ No newline at end of file diff --git a/charts/prometheus/am-opsgenie.sample.yaml b/charts/prometheus/am-opsgenie.sample.yaml new file mode 100644 index 0000000..3faf507 --- /dev/null +++ b/charts/prometheus/am-opsgenie.sample.yaml @@ -0,0 +1,58 @@ +# see: https://kb.vshn.ch/vshnsyn/how-tos/opsgenie.html +# see: https://prometheus.io/docs/alerting/latest/configuration/#opsgenie-receiver-%3Copsgenie_config%3E +alertmanager: + config: + global: + opsgenie_api_key: replace_api_key + route: + receiver: 'opsgenie' + routes: + - match: + alertname: Watchdog + receiver: "opsgenie" + - receiver: 'opsgenie' + receivers: + - name: 'opsgenie' + opsgenie_configs: + - priority: '{{ if eq .GroupLabels.severity "critical" }}P1{{ else if eq .GroupLabels.severity "warning" }}P2{{ else if eq .GroupLabels.severity "info" }}P3{{ else }}P4{{ end }}' + message: '[{{ .CommonLabels.tenant_id }}/{{ .CommonLabels.cluster_id }}] {{ .GroupLabels.alertname }} in {{ .GroupLabels.namespace }}' + description: |- + {{ if gt (len .Alerts.Firing) 0 -}} + Alerts Firing: + {{ range .Alerts.Firing }} + - Message: {{ .Annotations.message }} + Labels: + {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Annotations: + {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Source: {{ .GeneratorURL }} + {{ end }} + {{- end }} + {{ if gt (len .Alerts.Resolved) 0 -}} + Alerts Resolved: + {{ range .Alerts.Resolved }} + - Message: {{ .Annotations.message }} + Labels: + {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Annotations: + {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Source: {{ .GeneratorURL }} + {{ end }} + {{- end }} + details: + namespace: '{{- if .CommonLabels.exported_namespace -}}{{- .CommonLabels.exported_namespace -}}{{- else if .CommonLabels.namespace -}}{{- .CommonLabels.namespace -}}{{- end -}}' + pod: '{{- if .CommonLabels.pod -}}{{- .CommonLabels.pod -}}{{- end -}}' + deployment: '{{- if .CommonLabels.deployment -}}{{- .CommonLabels.deployment -}}{{- end -}}' + alertname: '{{ .GroupLabels.alertname }}' + cluster_id: '{{ .CommonLabels.cluster_id }}' + tenant_id: '{{ .CommonLabels.tenant_id }}' + severity: '{{ .GroupLabels.severity }}' + tags: '{{ .CommonLabels.tenant_id }}, + {{ .CommonLabels.cluster_id }}, + {{ .GroupLabels.severity }}, + {{ .GroupLabels.alertname }}, + {{ .GroupLabels.namespace }}, + {{- if .CommonLabels.exported_namespace -}}{{ .CommonLabels.exported_namespace }},{{- end -}}' + responders: + - id: replace_team_id + type: team diff --git a/charts/prometheus/am-opsgenie.yaml b/charts/prometheus/am-opsgenie.yaml new file mode 100644 index 0000000..3faf507 --- /dev/null +++ b/charts/prometheus/am-opsgenie.yaml @@ -0,0 +1,58 @@ +# see: https://kb.vshn.ch/vshnsyn/how-tos/opsgenie.html +# see: https://prometheus.io/docs/alerting/latest/configuration/#opsgenie-receiver-%3Copsgenie_config%3E +alertmanager: + config: + global: + opsgenie_api_key: replace_api_key + route: + receiver: 'opsgenie' + routes: + - match: + alertname: Watchdog + receiver: "opsgenie" + - receiver: 'opsgenie' + receivers: + - name: 'opsgenie' + opsgenie_configs: + - priority: '{{ if eq .GroupLabels.severity "critical" }}P1{{ else if eq .GroupLabels.severity "warning" }}P2{{ else if eq .GroupLabels.severity "info" }}P3{{ else }}P4{{ end }}' + message: '[{{ .CommonLabels.tenant_id }}/{{ .CommonLabels.cluster_id }}] {{ .GroupLabels.alertname }} in {{ .GroupLabels.namespace }}' + description: |- + {{ if gt (len .Alerts.Firing) 0 -}} + Alerts Firing: + {{ range .Alerts.Firing }} + - Message: {{ .Annotations.message }} + Labels: + {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Annotations: + {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Source: {{ .GeneratorURL }} + {{ end }} + {{- end }} + {{ if gt (len .Alerts.Resolved) 0 -}} + Alerts Resolved: + {{ range .Alerts.Resolved }} + - Message: {{ .Annotations.message }} + Labels: + {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Annotations: + {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} + {{ end }} Source: {{ .GeneratorURL }} + {{ end }} + {{- end }} + details: + namespace: '{{- if .CommonLabels.exported_namespace -}}{{- .CommonLabels.exported_namespace -}}{{- else if .CommonLabels.namespace -}}{{- .CommonLabels.namespace -}}{{- end -}}' + pod: '{{- if .CommonLabels.pod -}}{{- .CommonLabels.pod -}}{{- end -}}' + deployment: '{{- if .CommonLabels.deployment -}}{{- .CommonLabels.deployment -}}{{- end -}}' + alertname: '{{ .GroupLabels.alertname }}' + cluster_id: '{{ .CommonLabels.cluster_id }}' + tenant_id: '{{ .CommonLabels.tenant_id }}' + severity: '{{ .GroupLabels.severity }}' + tags: '{{ .CommonLabels.tenant_id }}, + {{ .CommonLabels.cluster_id }}, + {{ .GroupLabels.severity }}, + {{ .GroupLabels.alertname }}, + {{ .GroupLabels.namespace }}, + {{- if .CommonLabels.exported_namespace -}}{{ .CommonLabels.exported_namespace }},{{- end -}}' + responders: + - id: replace_team_id + type: team diff --git a/charts/prometheus/chart-prometheus-extras/.helmignore b/charts/prometheus/chart-prometheus-extras/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/prometheus/chart-prometheus-extras/Chart.yaml b/charts/prometheus/chart-prometheus-extras/Chart.yaml new file mode 100644 index 0000000..6cfa1e9 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: chart-extras +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/charts/prometheus/chart-prometheus-extras/README.md b/charts/prometheus/chart-prometheus-extras/README.md new file mode 100644 index 0000000..4386b21 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/README.md @@ -0,0 +1,12 @@ +# Prometheus extras Helm chart + +This chart deploys some extras for Prometheus: + +* Dashboards for Grafana: + * Custom Gatekeeper dashboard + * Helm exporter dashboard +* Gatekeeper monitoring requirements + * Service + * Servicemonitor +* Helm exporter requirements + * Servicemonitor \ No newline at end of file diff --git a/charts/prometheus/chart-prometheus-extras/templates/NOTES.txt b/charts/prometheus/chart-prometheus-extras/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/prometheus/chart-prometheus-extras/templates/_helpers.tpl b/charts/prometheus/chart-prometheus-extras/templates/_helpers.tpl new file mode 100644 index 0000000..d27e26f --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "chart-extras.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart-extras.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart-extras.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chart-extras.labels" -}} +helm.sh/chart: {{ include "chart-extras.chart" . }} +{{ include "chart-extras.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chart-extras.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart-extras.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chart-extras.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chart-extras.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/prometheus/chart-prometheus-extras/templates/cm-gatekeeper-dashboard.yaml b/charts/prometheus/chart-prometheus-extras/templates/cm-gatekeeper-dashboard.yaml new file mode 100644 index 0000000..e848d00 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/cm-gatekeeper-dashboard.yaml @@ -0,0 +1,2919 @@ +apiVersion: v1 +data: + gatekeeper-dashboard.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Visualize data from Gatekeeper", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15763, + "graphTooltip": 0, + "iteration": 1655971168244, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Summaries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 2, + "y": 1 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{` {{status}} `}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny constraints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 8, + "y": 1 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{` {{kind}} `}}", + "refId": "B" + } + ], + "title": "Total auidt violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{` {{kind}} `}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "Audit process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{` {{kind}} `}}", + "refId": "B" + } + ], + "title": "Total aduit violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Latency of audit operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 7 + }, + "id": 12, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_audit_duration_seconds_count[10m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit duration", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Description: Timestamp of last audit run time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 7 + }, + "id": 27, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "time() - gatekeeper_audit_last_run_time", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit last run time", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of watched GroupVersionKinds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 7 + }, + "id": 30, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of GroupVersionKinds with a registered watch intent", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 7 + }, + "id": 29, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds registered", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "interval": "", + "legendFormat": "{{` {{admission_status}} `}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate at wich the numbe of violations for constraints with \"deny\" enforcement action appear.\nMust be flat when it doesn't change, and a spike when it increases.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "irate(gatekeeper_violations{enforcement_action=\"deny\"}[5m])", + "interval": "", + "legendFormat": "{{` {{admission_status}} `}}", + "refId": "A" + } + ], + "title": "Audit deny violations change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 14, + "panels": [], + "title": "Controller", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{` {{status}} `}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 3, + "y": 18 + }, + "id": 19, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{` {{kind}} `}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Constraint Template ingestion duration distribution histogram\n\nWhen Gatekeeper is notified of a new constraint template, records the time from when it has loaded the resource from etcd to after OPA successfully returns from compiling the code. If there is a compilation error, the metric is not updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "gatekeeper-controller-manager-6bcfcf84cc-r4fjj" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 20, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_constraint_template_ingestion_count[10m])", + "interval": "", + "legendFormat": "{{` {{pod}} `}}", + "refId": "A" + } + ], + "title": "Constraint template ingestion duration", + "transformations": [], + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrains loaded per pod (audit and controller).\n\nAll pods should have the same number of constraints loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}", + "legendFormat": "{{` {{pod}} `}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrain templates loaded per pod (audit and controller).\n\nAll pods should have the same number of constraint templatess loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraint_templates{status=\"active\"}", + "legendFormat": "{{` {{pod}} `}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraints in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}[5m])", + "legendFormat": "{{` {{pod}} `}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints change rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraint templates in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraint_templates{status=\"active\"}[5m])", + "legendFormat": "{{` {{pod}} `}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 16, + "panels": [], + "title": "Admission webook", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller mutation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were \"mutated\" or not, or if you have deployed or not mutation policies.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_mutation_request_count[20m]))", + "interval": "", + "legendFormat": "{{` {{mutation_status}} `}}", + "refId": "A" + } + ], + "title": "Mutation request count rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller validation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were allowed or denied by the AC, or if you have deployed or not constraint or constraint templates.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_validation_request_count[20m]))", + "interval": "", + "legendFormat": "{{` {{mutation_status}} `}}", + "refId": "A" + } + ], + "title": "Validation request count rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 64, + "panels": [], + "title": "Constraints and constraint tempaltes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 47 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 3, + "y": 47 + }, + "id": 28, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{` {{status}} `}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 7, + "y": 47 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"dryrun\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 47 + }, + "id": 50, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"dryrun\"})", + "refId": "A" + } + ], + "title": "Active dryrun C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max constraints with status \"error\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 47 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"error\"})", + "refId": "A" + } + ], + "title": "Error C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 62, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{` {{kind}} `}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraint with enforcement action \"warn\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 50 + }, + "id": 51, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"warn\"})", + "refId": "A" + } + ], + "title": "Active warn C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"unrecognized\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 50 + }, + "id": 52, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"unrecognized\"})", + "refId": "A" + } + ], + "title": "Active unrecognized C", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 59, + "panels": [], + "title": "Alerts and deployed charts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "All Prometheus alert rules firing", + "gridPos": { + "h": 17, + "w": 9, + "x": 0, + "y": 54 + }, + "id": 57, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [ + "namespace" + ], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "inactive": false, + "noData": false, + "normal": false, + "pending": true + } + }, + "title": "Alerts firing", + "type": "alertlist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "dark-green", + "value": 0 + }, + { + "color": "blue", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIsoNoDateIfToday" + }, + { + "id": "custom.width", + "value": 197 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "auto" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "displayName", + "value": "State" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "UNKNOWN" + }, + "1": { + "index": 2, + "text": "DEPLOYED" + }, + "2": { + "index": 3, + "text": "DELETED" + }, + "3": { + "index": 4, + "text": "SUPERSEDED" + }, + "4": { + "index": 5, + "text": "?" + }, + "5": { + "index": 6, + "text": "DELETING" + }, + "6": { + "index": 7, + "text": "PENDING_INSTALL" + }, + "7": { + "index": 8, + "text": "PENDING_UPGRADE" + }, + "8": { + "index": 9, + "text": "PENDING_ROLLBACK" + }, + "-1": { + "index": 0, + "text": "FAILED" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version" + }, + "properties": [ + { + "id": "custom.width", + "value": 138 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "custom.width", + "value": 125 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "release" + }, + "properties": [ + { + "id": "custom.width", + "value": 258 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 182 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chart" + }, + "properties": [ + { + "id": "custom.width", + "value": 171 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 15, + "x": 9, + "y": 54 + }, + "id": 61, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "chart" + } + ] + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(helm_chart_info{chart=~\".*\", namespace=~\".*\", release=~\".*\"}) by (chart, release, namespace, version) != 2", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Helm releases", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, cluster)", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(kube_pod_info, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Gatekeeper custom dashboard", + "uid": "YBgRZG63", + "version": 3, + "weekStart": "" + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + grafana_dashboard: "1" + name: grafana-gatekeeper-dashboard + namespace: promstack diff --git a/charts/prometheus/chart-prometheus-extras/templates/cm-helm-exporter_rev2-dashboard.yaml b/charts/prometheus/chart-prometheus-extras/templates/cm-helm-exporter_rev2-dashboard.yaml new file mode 100644 index 0000000..c7b9a25 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/cm-helm-exporter_rev2-dashboard.yaml @@ -0,0 +1,260 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dashboard-helm-exporter + labels: + grafana_dashboard: "1" + namespace: promstack +data: + helm-exporter_rev2.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "Prometheus", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Helm stats exported to prometueus", + "editable": false, + "gnetId": 9367, + "graphTooltip": 0, + "id": 5, + "iteration": 1556325406207, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 21, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 5, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#629e51", + "#1f78c1" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + "0", + "2" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "Unknown", + "value": "0" + }, + { + "text": "Deployed", + "value": "1" + }, + { + "text": "Deleted", + "value": "2" + }, + { + "text": "Superseded", + "value": "3" + }, + { + "text": "Failed", + "value": "-1" + }, + { + "text": "Deleting", + "value": "5" + }, + { + "text": "Pending Install", + "value": "6" + }, + { + "text": "Pending Upgrade", + "value": "7" + }, + { + "text": "Pending Rollback", + "value": "8" + } + ] + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "min(helm_chart_info{chart=~\"$chart\", namespace=~\"$namespace\", release=~\"$release\"}) by (chart, release, namespace, version) != 2", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Helm Releases", + "transform": "table", + "type": "table" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "chart", + "options": [], + "query": "label_values(helm_chart_info, chart)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "release", + "options": [], + "query": "label_values(helm_chart_info, release)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(helm_chart_info, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Helm Exporter", + "uid": "Gqncyvfmz", + "version": 1 + } + diff --git a/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-service-metrics.yaml b/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-service-metrics.yaml new file mode 100644 index 0000000..5977984 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-service-metrics.yaml @@ -0,0 +1,13 @@ +kind: Service +apiVersion: v1 +metadata: + name: gatekeeper-metrics + labels: + app: gatekeeper + namespace: gatekeeper +spec: + selector: + app: gatekeeper + ports: + - name: metrics + port: 8888 diff --git a/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-servicemonitor.yaml b/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-servicemonitor.yaml new file mode 100644 index 0000000..3799ef2 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/gatekeeper-servicemonitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: gatekeeper + labels: + release: promstack + namespace: gatekeeper +spec: + selector: + matchLabels: + app: gatekeeper + endpoints: + - port: metrics + diff --git a/charts/prometheus/chart-prometheus-extras/templates/helm-exporter-servicemonitor.yaml b/charts/prometheus/chart-prometheus-extras/templates/helm-exporter-servicemonitor.yaml new file mode 100644 index 0000000..ca05d6e --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/helm-exporter-servicemonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: helm-exporter + # Change this to the namespace the Prometheus instance is running in + # namespace: default + labels: + release: promstack + namespace: helm-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: helm-exporter + endpoints: + - port: http diff --git a/charts/prometheus/chart-prometheus-extras/templates/tests/test-connection.yaml b/charts/prometheus/chart-prometheus-extras/templates/tests/test-connection.yaml new file mode 100644 index 0000000..b69dddf --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "chart-extras.fullname" . }}-test-connection" + labels: + {{- include "chart-extras.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "chart-extras.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/prometheus/chart-prometheus-extras/values.yaml b/charts/prometheus/chart-prometheus-extras/values.yaml new file mode 100644 index 0000000..42d4864 --- /dev/null +++ b/charts/prometheus/chart-prometheus-extras/values.yaml @@ -0,0 +1,82 @@ +# Default values for chart-extras. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/charts/prometheus/grafana/cm-gatekeeper-dashboard.yaml b/charts/prometheus/grafana/cm-gatekeeper-dashboard.yaml new file mode 100644 index 0000000..96aa10a --- /dev/null +++ b/charts/prometheus/grafana/cm-gatekeeper-dashboard.yaml @@ -0,0 +1,2919 @@ +apiVersion: v1 +data: + gatekeeper-dashboard.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Visualize data from Gatekeeper", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15763, + "graphTooltip": 0, + "iteration": 1655971168244, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Summaries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 2, + "y": 1 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny constraints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 8, + "y": 1 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Total auidt violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "Audit process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Total aduit violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Latency of audit operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 7 + }, + "id": 12, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_audit_duration_seconds_count[10m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit duration", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Description: Timestamp of last audit run time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 7 + }, + "id": 27, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "time() - gatekeeper_audit_last_run_time", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit last run time", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of watched GroupVersionKinds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 7 + }, + "id": 30, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of GroupVersionKinds with a registered watch intent", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 7 + }, + "id": 29, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds registered", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "interval": "", + "legendFormat": "{{admission_status}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate at wich the numbe of violations for constraints with \"deny\" enforcement action appear.\nMust be flat when it doesn't change, and a spike when it increases.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "irate(gatekeeper_violations{enforcement_action=\"deny\"}[5m])", + "interval": "", + "legendFormat": "{{admission_status}}", + "refId": "A" + } + ], + "title": "Audit deny violations change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 14, + "panels": [], + "title": "Controller", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 3, + "y": 18 + }, + "id": 19, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Constraint Template ingestion duration distribution histogram\n\nWhen Gatekeeper is notified of a new constraint template, records the time from when it has loaded the resource from etcd to after OPA successfully returns from compiling the code. If there is a compilation error, the metric is not updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "gatekeeper-controller-manager-6bcfcf84cc-r4fjj" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 20, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_constraint_template_ingestion_count[10m])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Constraint template ingestion duration", + "transformations": [], + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrains loaded per pod (audit and controller).\n\nAll pods should have the same number of constraints loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrain templates loaded per pod (audit and controller).\n\nAll pods should have the same number of constraint templatess loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraint_templates{status=\"active\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraints in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}[5m])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints change rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraint templates in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraint_templates{status=\"active\"}[5m])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 16, + "panels": [], + "title": "Admission webook", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller mutation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were \"mutated\" or not, or if you have deployed or not mutation policies.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_mutation_request_count[20m]))", + "interval": "", + "legendFormat": "{{mutation_status}}", + "refId": "A" + } + ], + "title": "Mutation request count rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller validation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were allowed or denied by the AC, or if you have deployed or not constraint or constraint templates.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_validation_request_count[20m]))", + "interval": "", + "legendFormat": "{{mutation_status}}", + "refId": "A" + } + ], + "title": "Validation request count rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 64, + "panels": [], + "title": "Constraints and constraint tempaltes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 47 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 3, + "y": 47 + }, + "id": 28, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 7, + "y": 47 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"dryrun\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 47 + }, + "id": 50, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"dryrun\"})", + "refId": "A" + } + ], + "title": "Active dryrun C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max constraints with status \"error\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 47 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"error\"})", + "refId": "A" + } + ], + "title": "Error C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 62, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraint with enforcement action \"warn\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 50 + }, + "id": 51, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"warn\"})", + "refId": "A" + } + ], + "title": "Active warn C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"unrecognized\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 50 + }, + "id": 52, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"unrecognized\"})", + "refId": "A" + } + ], + "title": "Active unrecognized C", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 59, + "panels": [], + "title": "Alerts and deployed charts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "All Prometheus alert rules firing", + "gridPos": { + "h": 17, + "w": 9, + "x": 0, + "y": 54 + }, + "id": 57, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [ + "namespace" + ], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "inactive": false, + "noData": false, + "normal": false, + "pending": true + } + }, + "title": "Alerts firing", + "type": "alertlist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "dark-green", + "value": 0 + }, + { + "color": "blue", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIsoNoDateIfToday" + }, + { + "id": "custom.width", + "value": 197 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "auto" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "displayName", + "value": "State" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "UNKNOWN" + }, + "1": { + "index": 2, + "text": "DEPLOYED" + }, + "2": { + "index": 3, + "text": "DELETED" + }, + "3": { + "index": 4, + "text": "SUPERSEDED" + }, + "4": { + "index": 5, + "text": "?" + }, + "5": { + "index": 6, + "text": "DELETING" + }, + "6": { + "index": 7, + "text": "PENDING_INSTALL" + }, + "7": { + "index": 8, + "text": "PENDING_UPGRADE" + }, + "8": { + "index": 9, + "text": "PENDING_ROLLBACK" + }, + "-1": { + "index": 0, + "text": "FAILED" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version" + }, + "properties": [ + { + "id": "custom.width", + "value": 138 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "custom.width", + "value": 125 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "release" + }, + "properties": [ + { + "id": "custom.width", + "value": 258 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 182 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chart" + }, + "properties": [ + { + "id": "custom.width", + "value": 171 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 15, + "x": 9, + "y": 54 + }, + "id": 61, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "chart" + } + ] + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(helm_chart_info{chart=~\".*\", namespace=~\".*\", release=~\".*\"}) by (chart, release, namespace, version) != 2", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Helm releases", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, cluster)", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(kube_pod_info, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Gatekeeper custom dashboard", + "uid": "YBgRZG63", + "version": 3, + "weekStart": "" + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + grafana_dashboard: "1" + name: grafana-gatekeeper-dashboard + namespace: promstack diff --git a/charts/prometheus/grafana/gatekeeper-dashboard.json b/charts/prometheus/grafana/gatekeeper-dashboard.json new file mode 100644 index 0000000..9042d35 --- /dev/null +++ b/charts/prometheus/grafana/gatekeeper-dashboard.json @@ -0,0 +1,2909 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Visualize data from Gatekeeper", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15763, + "graphTooltip": 0, + "iteration": 1655971168244, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "Summaries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 2, + "y": 1 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny constraints", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 8, + "y": 1 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Total auidt violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 10, + "panels": [], + "title": "Audit process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of constraint audit violations detected in the last audit cycle. \nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_violations)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Total aduit violations", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Latency of audit operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 7 + }, + "id": 12, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_audit_duration_seconds_count[10m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit duration", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Description: Timestamp of last audit run time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 7 + }, + "id": 27, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "time() - gatekeeper_audit_last_run_time", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Audit last run time", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of watched GroupVersionKinds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 7 + }, + "id": 30, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_watched_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of GroupVersionKinds with a registered watch intent", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 7 + }, + "id": 29, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"}) / sum(gatekeeper_watch_manager_intended_watch_gvk{pod=~\"gatekeeper-controller.+\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total GroupVersionKinds registered", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total violations for constarints with \"deny\" enforcement action detected by audit in time.\nIt doesn't include violations prevented by the admission controller. As such, most probably these belong to resources deployed before some constraint and constraint templates where in place.\n\nTo check audit violations, execute: \nkubectl describe \n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "gatekeeper_violations{enforcement_action=\"deny\"}", + "interval": "", + "legendFormat": "{{admission_status}}", + "refId": "A" + } + ], + "title": "Audit \"deny\" violations in time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate at wich the numbe of violations for constraints with \"deny\" enforcement action appear.\nMust be flat when it doesn't change, and a spike when it increases.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "irate(gatekeeper_violations{enforcement_action=\"deny\"}[5m])", + "interval": "", + "legendFormat": "{{admission_status}}", + "refId": "A" + } + ], + "title": "Audit deny violations change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 14, + "panels": [], + "title": "Controller", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 3, + "y": 18 + }, + "id": 19, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Constraint Template ingestion duration distribution histogram\n\nWhen Gatekeeper is notified of a new constraint template, records the time from when it has loaded the resource from etcd to after OPA successfully returns from compiling the code. If there is a compilation error, the metric is not updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "gatekeeper-controller-manager-6bcfcf84cc-r4fjj" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 20, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": true, + "expr": "rate(gatekeeper_constraint_template_ingestion_count[10m])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Constraint template ingestion duration", + "transformations": [], + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrains loaded per pod (audit and controller).\n\nAll pods should have the same number of constraints loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of constrain templates loaded per pod (audit and controller).\n\nAll pods should have the same number of constraint templatess loaded, but when they are modified, they may take some time to be on sync.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "gatekeeper_constraint_templates{status=\"active\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraints in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraints{enforcement_action=\"deny\", status=\"active\"}[5m])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraints change rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "If the number of constraint templates in audit and controller pods do not change, this graph is a line of zero value.\n\nWhen they increase or decrease, even for a short time, you will see a spike in this graph.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(gatekeeper_constraint_templates{status=\"active\"}[5m])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Constraint templates change rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 16, + "panels": [], + "title": "Admission webook", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller mutation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were \"mutated\" or not, or if you have deployed or not mutation policies.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_mutation_request_count[20m]))", + "interval": "", + "legendFormat": "{{mutation_status}}", + "refId": "A" + } + ], + "title": "Mutation request count rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of change for number of requests that are routed to all Gatekeeper admission controller validation webhook pods.\n\nThis includes all Kubernetes operations received for processing, no matter if they were allowed or denied by the AC, or if you have deployed or not constraint or constraint templates.\n\nTo see admision controller blocked violations, check the Kubernetes Audit Log, or check the Gatekeeper controller pods logs (this feature has to be enabled).\n\nTo check audit violations, execute: \nkubectl describe ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (mutation_status, pod) (rate(gatekeeper_validation_request_count[20m]))", + "interval": "", + "legendFormat": "{{mutation_status}}", + "refId": "A" + } + ], + "title": "Validation request count rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 64, + "panels": [], + "title": "Constraints and constraint tempaltes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"deny\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 47 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"deny\"})", + "refId": "A" + } + ], + "title": "Active deny C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraint templates for all controller pods.\n\nF.e.: 2 constraint templates x 3 controller pods = total 6", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 3, + "y": 47 + }, + "id": 28, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (gatekeeper_constraint_templates{pod=~\"gatekeeper-controller-manager-.+\"})", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Sum CT in controllers", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max different constraint templates on any pod.\nThis includes audit and controller pods.\n\nAs each controller pod syncs the number of constraint templates, the number each one have loaded can briefly change until sync finishes. This number represent how many constraint templates do have the pod that have more.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 7, + "y": 47 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"active\"})", + "refId": "A" + } + ], + "title": "Constraint Templates", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"dryrun\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 47 + }, + "id": 50, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"dryrun\"})", + "refId": "A" + } + ], + "title": "Active dryrun C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max constraints with status \"error\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 47 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraint_templates{status=\"error\"})", + "refId": "A" + } + ], + "title": "Error C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current number of constraints\nUnderstanding this metric: Gatekeeper subscribes to a watch on all constraint resources. Whenever one is created or deleted on the cluster, this count gets updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unrecognized" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dryrun" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deny" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 62, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (enforcement_action) (gatekeeper_constraints)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{kind}}", + "refId": "B" + } + ], + "title": "Sum Constraints", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraint with enforcement action \"warn\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 50 + }, + "id": 51, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"warn\"})", + "refId": "A" + } + ], + "title": "Active warn C", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Max active constraints with enforcement action \"unrecognized\" on all pods.\nThis includes audit and controller pods.\nMax implies different constraint templates in the cluster, instead of sum of the same constraint templates across different pods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 50 + }, + "id": 52, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(gatekeeper_constraints{status=\"active\",enforcement_action=\"unrecognized\"})", + "refId": "A" + } + ], + "title": "Active unrecognized C", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 59, + "panels": [], + "title": "Alerts and deployed charts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "All Prometheus alert rules firing", + "gridPos": { + "h": 17, + "w": 9, + "x": 0, + "y": 54 + }, + "id": 57, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [ + "namespace" + ], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "inactive": false, + "noData": false, + "normal": false, + "pending": true + } + }, + "title": "Alerts firing", + "type": "alertlist" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "dark-green", + "value": 0 + }, + { + "color": "blue", + "value": 2 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIsoNoDateIfToday" + }, + { + "id": "custom.width", + "value": 197 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "auto" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "displayName", + "value": "State" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "UNKNOWN" + }, + "1": { + "index": 2, + "text": "DEPLOYED" + }, + "2": { + "index": 3, + "text": "DELETED" + }, + "3": { + "index": 4, + "text": "SUPERSEDED" + }, + "4": { + "index": 5, + "text": "?" + }, + "5": { + "index": 6, + "text": "DELETING" + }, + "6": { + "index": 7, + "text": "PENDING_INSTALL" + }, + "7": { + "index": 8, + "text": "PENDING_UPGRADE" + }, + "8": { + "index": 9, + "text": "PENDING_ROLLBACK" + }, + "-1": { + "index": 0, + "text": "FAILED" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version" + }, + "properties": [ + { + "id": "custom.width", + "value": 138 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "custom.width", + "value": 125 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "release" + }, + "properties": [ + { + "id": "custom.width", + "value": 258 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 182 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chart" + }, + "properties": [ + { + "id": "custom.width", + "value": 171 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 15, + "x": 9, + "y": 54 + }, + "id": 61, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "chart" + } + ] + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(helm_chart_info{chart=~\".*\", namespace=~\".*\", release=~\".*\"}) by (chart, release, namespace, version) != 2", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Helm releases", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, cluster)", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(kube_pod_info, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Gatekeeper custom dashboard", + "uid": "YBgRZG63", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/prometheus/grafana/makefile b/charts/prometheus/grafana/makefile new file mode 100644 index 0000000..912b421 --- /dev/null +++ b/charts/prometheus/grafana/makefile @@ -0,0 +1,17 @@ +# https://grafana.com/grafana/dashboards/14828 +deploy_gatekeeper_dashboard_configmap: + kubectl -n promstack delete cm grafana-gatekeeper-dashboard ||: + kubectl -n promstack create cm grafana-gatekeeper-dashboard --from-file=./grafana/gatekeeper-dashboard.json + kubectl -n promstack label cm grafana-gatekeeper-dashboard grafana_dashboard=1 + +create_gatekeeper_dashboard_files: + @echo "Creating configmap:\n ./cm-gatekeeper-dashboard.yaml\nfrom dahsboard JSON file:\n ./gatekeeper-dashboard.json" + @kubectl -n promstack create cm grafana-gatekeeper-dashboard \ + --from-file=./gatekeeper-dashboard.json --dry-run=client -oyaml \ + | kubectl label -f- --dry-run=client -oyaml --local grafana_dashboard=1 \ + > ./cm-gatekeeper-dashboard.yaml + @echo "\nCreating chart template file:\n ../chart-prometheus-extras/templates/cm-gatekeeper-dashboard.yaml\nfrom configmap:\n ./cm-gatekeeper-dashboard.yaml" + @cat ./cm-gatekeeper-dashboard.yaml \ + | sed 's/{{/{{\` {{/g' \ + | sed 's/}}/}} \`}}/g' \ + > ../chart-prometheus-extras/templates/cm-gatekeeper-dashboard.yaml \ No newline at end of file diff --git a/charts/prometheus/makefile b/charts/prometheus/makefile new file mode 100644 index 0000000..dda51c0 --- /dev/null +++ b/charts/prometheus/makefile @@ -0,0 +1,19 @@ +show_audit_violations: + @echo "k8srequiredlabels.constraints.gatekeeper.sh/ns-must-have-gk" + @kubectl describe k8srequiredlabels.constraints.gatekeeper.sh/ns-must-have-gk | \ + grep -A999999 -e ' Violations:' + @echo "privilegedcontainers.constraints.gatekeeper.sh/privileged-container" + @kubectl describe privilegedcontainers.constraints.gatekeeper.sh/privileged-container | \ + grep -A999999 -e ' Violations:' + +log_gk_audit: + stern -n gatekeeper "gatekeeper-audit*" + +log_gk_controller_violations: + stern -n gatekeeper "gatekeeper-controller-*" | grep -i -E 'denied admission' + +log_kube_ac_denied: + minikube ssh -- tail /var/log/audit/audit.log -f | grep 'denied the request' + +log_kube_psa_denied: + minikube ssh -- tail /var/log/audit/audit.log -f | grep 'FailedCreate' \ No newline at end of file diff --git a/charts/prometheus/rules-gatekeeper.yaml b/charts/prometheus/rules-gatekeeper.yaml new file mode 100644 index 0000000..c0104a6 --- /dev/null +++ b/charts/prometheus/rules-gatekeeper.yaml @@ -0,0 +1,47 @@ +additionalPrometheusRules: +- name: gatekeeper + groups: + - name: gatekeeper + rules: + - alert: GatekeeperConstraintChange + annotations: + summary: 'Number of active Gatekeeper OPA deny constraints changed' + description: 'Number of active Gatekeeper OPA deny constraints changed. This can be the result of constraints wrongfully being deleted, or chaning the number of Gatekeeper pods, or cluster nodes. Please navigate to the runbook here for remediation steps: https://github.com/controlplaneio/eks-security-monitoring.' + runbook_url: https://github.com/controlplaneio/eks-security-monitoring + for: 1m + expr: irate(gatekeeper_constraints{enforcement_action="deny", status="active"}[5m])>0 + labels: + severity: High + - alert: GatekeeperConstraintTemplatesChange + annotations: + summary: 'Number of active Gatekeeper OPA deny constraints changed' + description: 'Number of active Gatekeeper OPA deny constraints changed. This can be the result of constraints wrongfully being deleted, or chaning the number of Gatekeeper pods, or cluster nodes. Please navigate to the runbook here for remediation steps: https://github.com/controlplaneio/eks-security-monitoring.' + runbook_url: https://github.com/controlplaneio/eks-security-monitoring + for: 1m + expr: irate(gatekeeper_constraint_templates{status="active"}[5m])>0 + labels: + severity: High + - alert: GatekeeperAuditViolationIncrease + annotations: + summary: 'Number of Gatekeeper audit violations have increased' + description: 'Number of Gatekeeper audit violations have increased. This may be possible for assets being created before a constarint was enable. Please navigate to the runbook here for remediation steps: https://github.com/controlplaneio/eks-security-monitoring.' + runbook_url: https://github.com/controlplaneio/eks-security-monitoring + for: 1m + expr: ((gatekeeper_violations - gatekeeper_violations offset 1m) > 0) + labels: + severity: High + - alert: GatekeeperWebhookStuck + annotations: + summary: 'Gatekeeper controller webhook for mutation or audit not receiving calls' + description: | + Gatekeeper admission controller pods webhook for mutation or audit not receiving calls. + They should be receiving a continuous stream of calls from the Kubernetes control plane for processing. + Please check the Gatekeeper dashboard for more information, the chart deployment status, that the service endpoint is deployed, and the number of pods of the deployment is correct. + + Please navigate to the runbook here for remediation steps: https://github.com/controlplaneio/eks-security-monitoring.' + runbook_url: https://github.com/controlplaneio/eks-security-monitoring + for: 1m + expr: ((gatekeeper_mutation_request_count - gatekeeper_mutation_request_count offset 1m) == 0) or ((gatekeeper_validation_request_count - gatekeeper_validation_request_count offset 1m) == 0) + labels: + severity: Medium + diff --git a/charts/robusta/am-robusta.yaml b/charts/robusta/am-robusta.yaml new file mode 100644 index 0000000..5fb964b --- /dev/null +++ b/charts/robusta/am-robusta.yaml @@ -0,0 +1,9 @@ +alertmanager: + route: + receiver: 'webhook' + config: + receivers: + - name: 'webhook' + webhook_configs: + - url: 'http://robusta-runner.robusta.svc.cluster.local/api/alerts' + send_resolved: true diff --git a/charts/robusta/generated_values.yaml b/charts/robusta/generated_values.yaml new file mode 100644 index 0000000..b528a45 --- /dev/null +++ b/charts/robusta/generated_values.yaml @@ -0,0 +1,23 @@ +globalConfig: + signing_key: replace_me + account_id: replace_me +sinksConfig: +- robusta_sink: + name: robusta_ui_sink + token: replace_me +clusterName: starter-k8s +enablePlatformPlaybooks: true +playbooksPersistentVolumeSize: 128Mi +kubewatch: + resources: + requests: + memory: 64Mi +grafanaRenderer: + resources: + requests: + memory: 64Mi +runner: + sendAdditionalTelemetry: false +rsa: + prv: 'replace_me' + pub: 'replace_me' diff --git a/charts/vault/makefile b/charts/vault/makefile new file mode 100644 index 0000000..12c8902 --- /dev/null +++ b/charts/vault/makefile @@ -0,0 +1,15 @@ + +all: setup_vault + +setup_vault: + kubectl exec vault-0 -- vault operator init \ + -key-shares=1 \ + -key-threshold=1 \ + -format=json > cluster-keys.json + jq -r ".unseal_keys_b64[]" cluster-keys.json + VAULT_UNSEAL_KEY=$$(jq -r ".unseal_keys_b64[]" cluster-keys.json) + kubectl exec vault-0 -- vault operator unseal $$VAULT_UNSEAL_KEY + kubectl exec -ti vault-1 -- vault operator raft join http://vault-0.vault-internal:8200 + kubectl exec -ti vault-2 -- vault operator raft join http://vault-0.vault-internal:8200 + kubectl exec -ti vault-1 -- vault operator unseal $$VAULT_UNSEAL_KEY + kubectl exec -ti vault-2 -- vault operator unseal $$VAULT_UNSEAL_KEY \ No newline at end of file diff --git a/charts/vault/values.yaml b/charts/vault/values.yaml new file mode 100644 index 0000000..7e6e7e8 --- /dev/null +++ b/charts/vault/values.yaml @@ -0,0 +1,14 @@ + +server: + affinity: "" + ha: + enabled: true + raft: + enabled: true + +injector: + agentDefaults: + cpuLimit: 550m + cpuRequest: 300m + memLimit: 150Mi + memRequest: 70Mi \ No newline at end of file diff --git a/cluster/eks/eksctl-cluster.yaml b/cluster/eks/eksctl-cluster.yaml new file mode 100644 index 0000000..c68c061 --- /dev/null +++ b/cluster/eks/eksctl-cluster.yaml @@ -0,0 +1,15 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: eks-test-cluster + region: eu-north-1 + +nodeGroups: + - name: ng-eks-test-cluster-1 + instanceType: t3.medium + desiredCapacity: 2 + +cloudWatch: + clusterLogging: + enableTypes: ["*"] diff --git a/cluster/eks/makefile b/cluster/eks/makefile new file mode 100644 index 0000000..9d7a160 --- /dev/null +++ b/cluster/eks/makefile @@ -0,0 +1,28 @@ + +start: + @echo "If you get errors from awscli and Kubectl version, get an older one:" + @echo "curl -LO \"https://dl.k8s.io/release/v1.23.6/bin/linux/amd64/kubectl\"" + eksctl create cluster -f eksctl-cluster.yaml + +delete: + # Delete services with external ip + # kubectl delete svc + # Delete pod's disruption budget + # kubectl delete poddisruptionbudget {name} -n {namespace} + # Delete nodegroup ignoring pod disruption budget + eksctl delete nodegroup --wait --cluster=eks-test-cluster --name=ng-eks-test-cluster-1 --drain=false --region=eu-north-1 + # Delete cluster + eksctl delete cluster --wait -f eksctl-cluster.yaml + + +enable_audit_log: + @echo "Audit log already enabled on EKS" + +kubeconfig: + aws eks update-kubeconfig --name eks-test-cluster + +audit_log: + aws logs tail "/aws/eks/eks-test-cluster/cluster" --follow + +audit_log_pss: + aws logs tail "/aws/eks/eks-test-cluster/cluster" --follow "FailedCreate" \ No newline at end of file diff --git a/cluster/minikube/falco/audit-policy.yaml b/cluster/minikube/falco/audit-policy.yaml new file mode 100644 index 0000000..25b8fd0 --- /dev/null +++ b/cluster/minikube/falco/audit-policy.yaml @@ -0,0 +1,68 @@ +apiVersion: audit.k8s.io/v1 # This is required. +kind: Policy +# Don't generate audit events for all requests in RequestReceived stage. +omitStages: + - "RequestReceived" +rules: + # Log pod changes at RequestResponse level + - level: RequestResponse + resources: + - group: "" + # Resource "pods" doesn't match requests to any subresource of pods, + # which is consistent with the RBAC policy. + resources: ["pods"] + # Log "pods/log", "pods/status" at Metadata level + - level: Metadata + resources: + - group: "" + resources: ["pods/log", "pods/status"] + + # Don't log requests to a configmap called "controller-leader" + - level: None + resources: + - group: "" + resources: ["configmaps"] + resourceNames: ["controller-leader"] + + # Don't log watch requests by the "system:kube-proxy" on endpoints or services + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core API group + resources: ["endpoints", "services"] + + # Don't log authenticated requests to certain non-resource URL paths. + - level: None + userGroups: ["system:authenticated"] + nonResourceURLs: + - "/api*" # Wildcard matching. + - "/version" + + # Log the request body of configmap changes in kube-system. + - level: Request + resources: + - group: "" # core API group + resources: ["configmaps"] + # This rule only applies to resources in the "kube-system" namespace. + # The empty string "" can be used to select non-namespaced resources. + namespaces: ["kube-system"] + + # Log configmap and secret changes in all other namespaces at the Metadata level. + - level: Metadata + resources: + - group: "" # core API group + resources: ["secrets", "configmaps"] + + # Log all other resources in core and extensions at the Request level. + - level: Request + resources: + - group: "" # core API group + - group: "extensions" # Version of group should NOT be included. + + # A catch-all rule to log all other requests at the Metadata level. + - level: Metadata + # Long-running requests like watches that fall under this rule will not + # generate an audit event in RequestReceived. + omitStages: + - "RequestReceived" diff --git a/cluster/minikube/falco/falco_service.yaml b/cluster/minikube/falco/falco_service.yaml new file mode 100644 index 0000000..9f4221f --- /dev/null +++ b/cluster/minikube/falco/falco_service.yaml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: falco + labels: + app: falco +spec: + type: NodePort + selector: + app: falco + ports: + - protocol: TCP + port: 8765 + targetPort: 8765 + nodePort: 30545 diff --git a/cluster/minikube/falco/kube-apiserver-patch-falco.diff b/cluster/minikube/falco/kube-apiserver-patch-falco.diff new file mode 100644 index 0000000..6b9c821 --- /dev/null +++ b/cluster/minikube/falco/kube-apiserver-patch-falco.diff @@ -0,0 +1,46 @@ +--- kube-apiserver-original.yaml 2022-06-06 13:02:25.044395565 +0200 ++++ kube-apiserver-modified.yaml 2022-06-06 13:07:29.129971834 +0200 +@@ -23,6 +23,8 @@ + - --etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt + - --etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key + - --etcd-servers=https://127.0.0.1:2379 ++ - --audit-policy-file=/var/lib/k8s_audit/audit-policy.yaml ++ - --audit-log-path=/var/log/audit/audit.log ++ - --audit-webhook-config-file=/var/lib/k8s_audit/webhook-config.yaml + - --kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt + - --kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname +@@ -85,12 +87,26 @@ + - mountPath: /usr/share/ca-certificates + name: usr-share-ca-certificates + readOnly: true ++ - mountPath: /var/lib/k8s_audit/audit-policy.yaml ++ name: audit ++ readOnly: true ++ - mountPath: /var/lib/k8s_audit/webhook-config.yaml ++ name: audit-webhook ++ readOnly: true ++ - mountPath: /var/log/audit/audit.log ++ name: audit-log ++ readOnly: false + hostNetwork: true + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + volumes: ++ - name: audit ++ hostPath: ++ path: /var/lib/k8s_audit/audit-policy.yaml ++ type: File ++ - name: audit ++ hostPath: ++ path: /var/lib/k8s_audit/webhook-config.yaml ++ type: File ++ - name: audit-log ++ hostPath: ++ path: /var/log/audit/audit.log ++ type: FileOrCreate + - hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate diff --git a/cluster/minikube/falco/kube-apiserver-patch.diff b/cluster/minikube/falco/kube-apiserver-patch.diff new file mode 100644 index 0000000..ee278c4 --- /dev/null +++ b/cluster/minikube/falco/kube-apiserver-patch.diff @@ -0,0 +1,38 @@ +--- kube-apiserver-original.yaml 2022-06-06 13:02:25.044395565 +0200 ++++ kube-apiserver-modified.yaml 2022-06-06 13:07:29.129971834 +0200 +@@ -23,6 +23,8 @@ + - --etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt + - --etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key + - --etcd-servers=https://127.0.0.1:2379 ++ - --audit-policy-file=/var/lib/k8s_audit/audit-policy.yaml ++ - --audit-log-path=/var/log/audit/audit.log + - --kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt + - --kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname +@@ -85,12 +87,26 @@ + - mountPath: /usr/share/ca-certificates + name: usr-share-ca-certificates + readOnly: true ++ - mountPath: /var/lib/k8s_audit/audit-policy.yaml ++ name: audit ++ readOnly: true ++ - mountPath: /var/log/audit/audit.log ++ name: audit-log ++ readOnly: false + hostNetwork: true + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + volumes: ++ - name: audit ++ hostPath: ++ path: /var/lib/k8s_audit/audit-policy.yaml ++ type: File ++ - name: audit-log ++ hostPath: ++ path: /var/log/audit/audit.log ++ type: FileOrCreate + - hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate diff --git a/cluster/minikube/falco/minikube_enable_audit_log_falco.sh b/cluster/minikube/falco/minikube_enable_audit_log_falco.sh new file mode 100755 index 0000000..bc36952 --- /dev/null +++ b/cluster/minikube/falco/minikube_enable_audit_log_falco.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# https://medium.com/trendyol-tech/how-to-enable-kubernetes-auditing-feature-in-minikube-running-locally-e45f0d68ff82 + +set -e + +echo "Enabling Kubernetes Audit Log on Minikube" + +# curl -sLO https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml + +# minikube ssh "curl -sLO https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml" +minikube cp audit-policy.yaml minikube:/home/docker/audit-policy.yaml +minikube cp webhook-config.yaml minikube:/home/docker/webhook-config.yaml +minikube ssh "sudo mkdir -p /var/lib/k8s_audit /var/log/audit" +minikube ssh "sudo mv audit-policy.yaml /var/lib/k8s_audit" +minikube cp kube-apiserver-patch.diff minikube:/home/docker/kube-apiserver-patch.diff +minikube ssh "sudo patch /etc/kubernetes/manifests/kube-apiserver.yaml < kube-apiserver-patch.diff" + +echo "Waiting for control plane to reboot and audit.log file to have content" +minikube ssh 'while ! [ -s /var/log/audit/audit.log ]; do echo -n "." ; sleep 3 ; done' +echo "" +echo "Audit log enabled" diff --git a/cluster/minikube/falco/webhook-config.yaml b/cluster/minikube/falco/webhook-config.yaml new file mode 100644 index 0000000..9a538ab --- /dev/null +++ b/cluster/minikube/falco/webhook-config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Config +clusters: +- name: falco + cluster: + server: http://127.0.0.1:30545/k8s-audit +contexts: +- context: + cluster: falco + user: "" + name: default-context +current-context: default-context +preferences: {} +users: [] \ No newline at end of file diff --git a/cluster/minikube/makefile b/cluster/minikube/makefile new file mode 100644 index 0000000..27733be --- /dev/null +++ b/cluster/minikube/makefile @@ -0,0 +1,27 @@ + + +# See https://github.com/prometheus-operator/kube-prometheus +start: + minikube delete ||: + minikube start \ + --kubernetes-version=v1.23.0 --memory=6g --cpus 4 \ + --bootstrapper=kubeadm \ + --extra-config=kubelet.authentication-token-webhook=true \ + --extra-config=kubelet.authorization-mode=Webhook \ + --extra-config=scheduler.bind-address=0.0.0.0 \ + --extra-config=controller-manager.bind-address=0.0.0.0 \ + --extra-config=kubelet.housekeeping-interval=10s + minikube addons enable metrics-server + +kubeconfig: + minikube update-context + +delete: + minikube delete + +enable_audit_log: + kubectl apply -f ./falco/falco_service.yaml -n falco --wait + @cd falco && ./minikube_enable_audit_log_falco.sh + +audit_log: + minikube ssh -- tail /var/log/audit/audit.log -f \ No newline at end of file diff --git a/exercises/gatekeeper/c-privileged-container-tampered.yaml b/exercises/gatekeeper/c-privileged-container-tampered.yaml new file mode 100644 index 0000000..8d778b1 --- /dev/null +++ b/exercises/gatekeeper/c-privileged-container-tampered.yaml @@ -0,0 +1,10 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: PrivilegedContainers +metadata: + name: psp-privileged-container +spec: + enforcementAction: dryrun + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/exercises/gatekeeper/makefile b/exercises/gatekeeper/makefile new file mode 100644 index 0000000..5a19916 --- /dev/null +++ b/exercises/gatekeeper/makefile @@ -0,0 +1,11 @@ +delete_constraints-helm: + helm delete constraints -n gatekeeper + +deploy_privileged_tenant: + kubectl apply -f ./privileged-pod.yaml -n tenant + +delete_privileged_tenant: + kubectl delete .f ./privileged-pod.yaml -n tenant + +tamper_privileged_constraint: + kubectl apply -f ./c-privileged-container-tampered.yaml \ No newline at end of file diff --git a/exercises/gatekeeper/privileged-pod.yaml b/exercises/gatekeeper/privileged-pod.yaml new file mode 100644 index 0000000..0417a60 --- /dev/null +++ b/exercises/gatekeeper/privileged-pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + run: privileged + name: privileged +spec: + containers: + - image: busybox + name: privileged + command: ['sh', '-c', 'sleep 99999'] + securityContext: + privileged: true diff --git a/exercises/hpa/makefile b/exercises/hpa/makefile new file mode 100644 index 0000000..5e417df --- /dev/null +++ b/exercises/hpa/makefile @@ -0,0 +1,34 @@ + +# HPA testing +# https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/ + +hpa-deployment-test: + kubectl apply -f rbac/hpa/test-deployment.yaml + +hpa-create: + kubectl delete hpa php-apache ||: + kubectl autoscale deployment php-apache --cpu-percent=20 --min=1 --max=10 + +hpa-rbac: + kubectl apply -f rbac/hpa/hpa-role.yaml + kubectl apply -f rbac/hpa/hpa-rolebinding.yaml + +hpa-describe: + watch -n 3 kubectl describe hpa/php-apache + +hpa-watch-top: + watch -n 3 kubectl top pods + +hpa-load: + kubectl run -it load-generator --rm --image=busybox:1.28 --restart=Never -- /bin/sh -c "while sleep 0.01; do wget -q -O- http://php-apache; done" + +# Load using external k6 client + +hpa-expose: + echo "You have to run `minikube tunnel` on a separate terminal" + kubectl expose deployment php-apache --type=LoadBalancer --name=apache + minikube service apache --url + + +hpa-k6-load: + k6 run ./rbac/hpa/k6.js \ No newline at end of file diff --git a/exercises/jobs/exampleCronJob.yaml.in b/exercises/jobs/exampleCronJob.yaml.in new file mode 100644 index 0000000..0358b1a --- /dev/null +++ b/exercises/jobs/exampleCronJob.yaml.in @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: sec-test-$NAME +spec: + schedule: "*/$FREQ * * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: + cronjob: sec-test-$NAME + spec: + backoffLimit: 3 + template: + metadata: + labels: + cronjob: sec-test-$NAME + spec: + containers: + - name: random-success + image: bash + command: ["bash"] + args: [ '-c', 'echo "Random fail or success" ; if [ "$RANDOM" -gt "$CHANCE" ]; then echo "fail" && exit 1; else echo "success"; fi' ] + restartPolicy: Never diff --git a/exercises/jobs/exampleJob.yaml.in b/exercises/jobs/exampleJob.yaml.in new file mode 100644 index 0000000..a5bca62 --- /dev/null +++ b/exercises/jobs/exampleJob.yaml.in @@ -0,0 +1,15 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: sec-test-$JOBNAME- +spec: + backoffLimit: 4 + template: + spec: + restartPolicy: Never + containers: + - name: random-success + image: bash + command: ["bash"] + args: [ '-c', 'echo "Random fail or success" ; if [ "$RANDOM" -gt "5000" ]; then echo "fail" && exit 1; else echo "success"; fi' ] + diff --git a/exercises/jobs/makefile b/exercises/jobs/makefile new file mode 100644 index 0000000..85b5bcb --- /dev/null +++ b/exercises/jobs/makefile @@ -0,0 +1,33 @@ + +create_cronjobs: + @$(MAKE) -s create_namespace NAMESPACE=tennant + @$(MAKE) -s cronjob_single NAME=psp CHANCE=5000 FREQ=5 + @$(MAKE) -s cronjob_single NAME=netpolicy CHANCE=8500 FREQ=5 + @$(MAKE) -s cronjob_single NAME=admission CHANCE=8500 FREQ=5 + +delete_cronjobs: + kubectl delete cronjob sec-test-psp -n tennant + kubectl delete cronjob sec-test-netpolicy -n tennant + kubectl delete cronjob sec-test-admission -n tennant + +create_mock_jobs: + @echo "This will create some mock Kubernetes jobs spaced minutes time" + @$(MAKE) -s create_namespace NAMESPACE=tennant + @$(MAKE) -s job_single NAME=psp + @$(MAKE) -s job_single NAME=netpolicy + @$(MAKE) -s job_single NAME=admission + +job_single: + @command -v envsubst > /dev/null || ( echo "envsubst command required" && exit 1 ) + @echo "Launching job $$NAME" + cat ./exampleJob.yaml.in | \ + NAME=$$NAME envsubst '$$NAME' | \ + kubectl create -n tennant -f - + +cronjob_single: + cat ./exampleCronJob.yaml.in | \ + NAME=$$NAME CHANCE=$$CHANCE FREQ=$$FREQ envsubst '$$NAME $$CHANCE $$FREQ' | \ + kubectl apply -n tennant -f - + +create_namespace: + @kubectl create namespace "$$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - diff --git a/exercises/psp/check_pss_levels.sh b/exercises/psp/check_pss_levels.sh new file mode 100755 index 0000000..66ab3a9 --- /dev/null +++ b/exercises/psp/check_pss_levels.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +echo "" > test_psa.log + +ALL_NAMESPACES=$(kubectl get namespaces -o name) + +for NAMESPACE in $ALL_NAMESPACES +do + + echo "Checking namespace: $NAMESPACE" | tee -a test_psa.log + + RECOMMENDED="restricted" + LEVEL="restricted" + RESTRICTED_OUTPUT=$(kubectl label --dry-run=server --overwrite "$NAMESPACE" pod-security.kubernetes.io/enforce=$LEVEL 2>&1 > /dev/null) + if [ "$RESTRICTED_OUTPUT" == "" ]; then + echo " Restricted: ok" >> test_psa.log + else + echo " Restricted: warnings" >> test_psa.log + echo "$RESTRICTED_OUTPUT" >> test_psa.log + RECOMMENDED="baseline" + fi + + LEVEL="baseline" + BASELINE_OUTPUT=$(kubectl label --dry-run=server --overwrite "$NAMESPACE" pod-security.kubernetes.io/enforce=$LEVEL 2>&1 > /dev/null) + if [ "$BASELINE_OUTPUT" == "" ]; then + echo " Baseline: ok" >> test_psa.log + else + echo " Baseline: warnings" >> test_psa.log + echo "$BASELINE_OUTPUT" >> test_psa.log + RECOMMENDED="privileged" + fi + + echo " Recommended level: $RECOMMENDED" | tee -a test_psa.log + echo "-------------------------------------------------------------------" >> test_psa.log + +done diff --git a/exercises/psp/privileged-deployment.yaml b/exercises/psp/privileged-deployment.yaml new file mode 100644 index 0000000..beef906 --- /dev/null +++ b/exercises/psp/privileged-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: privileged-deployment + name: privileged-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: privileged-deployment + template: + metadata: + labels: + app: privileged-deployment + spec: + containers: + - image: busybox + imagePullPolicy: Always + name: busybox + command: ['sh', '-c', 'sleep 99999'] + securityContext: + privileged: true + terminationGracePeriodSeconds: 3 \ No newline at end of file diff --git a/exercises/psp/psa_denied.json b/exercises/psp/psa_denied.json new file mode 100644 index 0000000..b4b4f39 --- /dev/null +++ b/exercises/psp/psa_denied.json @@ -0,0 +1,670 @@ +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "ed7c77a7-57aa-4309-99a7-5b32042a12ae", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54f363faa9", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54f363faa9", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "840" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-dtbmx\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.478451Z", + "stageTimestamp": "2022-07-13T15:32:29.485850Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "1eb8337f-da91-41de-9873-9bd2534c0a6c", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54f42f4e5a", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54f42f4e5a", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-rcptm\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.491264Z", + "stageTimestamp": "2022-07-13T15:32:29.498700Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "4d248f58-0f20-426b-8acf-56bd04d79840", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54f4b6a3f0", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54f4b6a3f0", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-mp764\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.501273Z", + "stageTimestamp": "2022-07-13T15:32:29.504754Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "5f80e2d4-7cd4-4a26-ace6-9efc986141b1", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54f62eaf8c", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54f62eaf8c", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-sz5dz\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.524630Z", + "stageTimestamp": "2022-07-13T15:32:29.527131Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "2fb85bca-7858-47ac-87b6-c47ccabd074f", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54f8d9ebc2", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54f8d9ebc2", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-9xpv9\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.569318Z", + "stageTimestamp": "2022-07-13T15:32:29.572003Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "c3a346b6-32ee-42fd-979c-43fbaf4ab77f", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d54fdf65142", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d54fdf65142", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-q9fdn\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.655089Z", + "stageTimestamp": "2022-07-13T15:32:29.658039Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "266dc0a1-c1d7-45e8-bfbe-cf07fbdad329", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d5507bd2859", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d5507bd2859", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-lj8ck\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:29Z", + "lastTimestamp": "2022-07-13T15:32:29Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:29.819087Z", + "stageTimestamp": "2022-07-13T15:32:29.821625Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "f38b1dc9-f20b-4577-af9e-9650b84320d5", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d551b0ffa44", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d551b0ffa44", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-pfbrg\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:30Z", + "lastTimestamp": "2022-07-13T15:32:30Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:30.143381Z", + "stageTimestamp": "2022-07-13T15:32:30.145829Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "19b44360-80d2-405d-a829-2b25ed913372", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d554182c719", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d554182c719", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "Error creating: pods \"helm-exporter-c5967955c-pvbdp\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:30Z", + "lastTimestamp": "2022-07-13T15:32:30Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:30.788152Z", + "stageTimestamp": "2022-07-13T15:32:30.790048Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} +{ + "kind": "Event", + "apiVersion": "audit.k8s.io/v1", + "level": "Request", + "auditID": "5b917fc2-bb4b-47e2-8d10-c35b8a7bacc5", + "stage": "ResponseComplete", + "requestURI": "/api/v1/namespaces/test-restricted/events", + "verb": "create", + "user": { + "username": "system:serviceaccount:kube-system:replicaset-controller", + "uid": "4918e1ac-0690-45dd-a93a-d08d266480b4", + "groups": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ] + }, + "sourceIPs": [ + "192.168.39.162" + ], + "userAgent": "kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller", + "objectRef": { + "resource": "events", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c.17016d558e4e6398", + "apiVersion": "v1" + }, + "responseStatus": { + "metadata": {}, + "code": 201 + }, + "requestObject": { + "kind": "Event", + "apiVersion": "v1", + "metadata": { + "name": "helm-exporter-c5967955c.17016d558e4e6398", + "namespace": "test-restricted", + "creationTimestamp": null + }, + "involvedObject": { + "kind": "ReplicaSet", + "namespace": "test-restricted", + "name": "helm-exporter-c5967955c", + "uid": "79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77", + "apiVersion": "apps/v1", + "resourceVersion": "845" + }, + "reason": "FailedCreate", + "message": "(combined from similar events): Error creating: pods \"helm-exporter-c5967955c-jfbfl\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")", + "source": { + "component": "replicaset-controller" + }, + "firstTimestamp": "2022-07-13T15:32:32Z", + "lastTimestamp": "2022-07-13T15:32:32Z", + "count": 1, + "type": "Warning", + "eventTime": null, + "reportingComponent": "", + "reportingInstance": "" + }, + "requestReceivedTimestamp": "2022-07-13T15:32:32.076476Z", + "stageTimestamp": "2022-07-13T15:32:32.078351Z", + "annotations": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\"" + } +} \ No newline at end of file diff --git a/exercises/psp/psa_denied.txt b/exercises/psp/psa_denied.txt new file mode 100644 index 0000000..0e2870c --- /dev/null +++ b/exercises/psp/psa_denied.txt @@ -0,0 +1,11 @@ +minikube ssh -- tail /var/log/audit/audit.log -f | grep 'FailedCreate' +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"ed7c77a7-57aa-4309-99a7-5b32042a12ae","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54f363faa9","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54f363faa9","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"840"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-dtbmx\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.478451Z","stageTimestamp":"2022-07-13T15:32:29.485850Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"1eb8337f-da91-41de-9873-9bd2534c0a6c","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54f42f4e5a","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54f42f4e5a","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-rcptm\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.491264Z","stageTimestamp":"2022-07-13T15:32:29.498700Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"4d248f58-0f20-426b-8acf-56bd04d79840","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54f4b6a3f0","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54f4b6a3f0","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-mp764\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.501273Z","stageTimestamp":"2022-07-13T15:32:29.504754Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"5f80e2d4-7cd4-4a26-ace6-9efc986141b1","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54f62eaf8c","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54f62eaf8c","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-sz5dz\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.524630Z","stageTimestamp":"2022-07-13T15:32:29.527131Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"2fb85bca-7858-47ac-87b6-c47ccabd074f","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54f8d9ebc2","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54f8d9ebc2","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-9xpv9\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.569318Z","stageTimestamp":"2022-07-13T15:32:29.572003Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"c3a346b6-32ee-42fd-979c-43fbaf4ab77f","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d54fdf65142","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d54fdf65142","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-q9fdn\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.655089Z","stageTimestamp":"2022-07-13T15:32:29.658039Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"266dc0a1-c1d7-45e8-bfbe-cf07fbdad329","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d5507bd2859","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d5507bd2859","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-lj8ck\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:29Z","lastTimestamp":"2022-07-13T15:32:29Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:29.819087Z","stageTimestamp":"2022-07-13T15:32:29.821625Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"f38b1dc9-f20b-4577-af9e-9650b84320d5","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d551b0ffa44","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d551b0ffa44","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-pfbrg\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:30Z","lastTimestamp":"2022-07-13T15:32:30Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:30.143381Z","stageTimestamp":"2022-07-13T15:32:30.145829Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"19b44360-80d2-405d-a829-2b25ed913372","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d554182c719","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d554182c719","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"Error creating: pods \"helm-exporter-c5967955c-pvbdp\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:30Z","lastTimestamp":"2022-07-13T15:32:30Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:30.788152Z","stageTimestamp":"2022-07-13T15:32:30.790048Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} +{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"5b917fc2-bb4b-47e2-8d10-c35b8a7bacc5","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/test-restricted/events","verb":"create","user":{"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"4918e1ac-0690-45dd-a93a-d08d266480b4","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["192.168.39.162"],"userAgent":"kube-controller-manager/v1.23.0 (linux/amd64) kubernetes/ab69524/system:serviceaccount:kube-system:replicaset-controller","objectRef":{"resource":"events","namespace":"test-restricted","name":"helm-exporter-c5967955c.17016d558e4e6398","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":201},"requestObject":{"kind":"Event","apiVersion":"v1","metadata":{"name":"helm-exporter-c5967955c.17016d558e4e6398","namespace":"test-restricted","creationTimestamp":null},"involvedObject":{"kind":"ReplicaSet","namespace":"test-restricted","name":"helm-exporter-c5967955c","uid":"79ec73fd-7e9c-4f0f-ab4d-c642d90f2d77","apiVersion":"apps/v1","resourceVersion":"845"},"reason":"FailedCreate","message":"(combined from similar events): Error creating: pods \"helm-exporter-c5967955c-jfbfl\" is forbidden: violates PodSecurity \"restricted:latest\": allowPrivilegeEscalation != false (container \"helm-exporter\" must set securityContext.allowPrivilegeEscalation=false), unrestricted capabilities (container \"helm-exporter\" must set securityContext.capabilities.drop=[\"ALL\"]), runAsNonRoot != true (pod or container \"helm-exporter\" must set securityContext.runAsNonRoot=true), seccompProfile (pod or container \"helm-exporter\" must set securityContext.seccompProfile.type to \"RuntimeDefault\" or \"Localhost\")","source":{"component":"replicaset-controller"},"firstTimestamp":"2022-07-13T15:32:32Z","lastTimestamp":"2022-07-13T15:32:32Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""},"requestReceivedTimestamp":"2022-07-13T15:32:32.076476Z","stageTimestamp":"2022-07-13T15:32:32.078351Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:controller:replicaset-controller\" of ClusterRole \"system:controller:replicaset-controller\" to ServiceAccount \"replicaset-controller/kube-system\""}} diff --git a/exercises/psp/psp-hostpath.yaml b/exercises/psp/psp-hostpath.yaml new file mode 100644 index 0000000..40b0899 --- /dev/null +++ b/exercises/psp/psp-hostpath.yaml @@ -0,0 +1,56 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp-hostpath +spec: + allowedHostPaths: + - pathPrefix: "/foo" + # readOnly: false + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostPID: false + hostIPC: false + seLinux: { 'rule': 'RunAsAny' } + runAsUser: { 'rule': 'RunAsAny' } + fsGroup: { 'rule': 'RunAsAny' } + supplementalGroups: { 'rule': 'RunAsAny' } + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cr-hostpath +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - psp-hostpath + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: crb-hostpath +roleRef: + kind: ClusterRole + name: cr-hostpath + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: sa-hostpath + namespace: developers + diff --git a/exercises/psp/psp-nonroot.yaml b/exercises/psp/psp-nonroot.yaml new file mode 100644 index 0000000..ecf65af --- /dev/null +++ b/exercises/psp/psp-nonroot.yaml @@ -0,0 +1,24 @@ +# https://docs.mirantis.com/mke/3.5/ops/deploy-apps-k8s/pod-security-policies/psp-examples.html + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: norootcontainers +spec: + allowPrivilegeEscalation: false + allowedHostPaths: + - pathPrefix: /dev/null + readOnly: true + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' diff --git a/exercises/psp/psp-privileged.yaml b/exercises/psp/psp-privileged.yaml new file mode 100644 index 0000000..bbfd348 --- /dev/null +++ b/exercises/psp/psp-privileged.yaml @@ -0,0 +1,29 @@ +# https://kubernetes.io/docs/concepts/security/pod-security-policy/ + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + volumes: + - '*' + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' diff --git a/exercises/psp/psp-restricted.yaml b/exercises/psp/psp-restricted.yaml new file mode 100644 index 0000000..7bdef1e --- /dev/null +++ b/exercises/psp/psp-restricted.yaml @@ -0,0 +1,49 @@ +# https://kubernetes.io/docs/concepts/security/pod-security-policy/ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted + annotations: + # docker/default identifies a profile for seccomp, but it is not particularly tied to the Docker runtime + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false \ No newline at end of file diff --git a/exercises/psp/psp-seccomppolicy.yaml b/exercises/psp/psp-seccomppolicy.yaml new file mode 100644 index 0000000..bbae95b --- /dev/null +++ b/exercises/psp/psp-seccomppolicy.yaml @@ -0,0 +1,27 @@ +# https://docs.mirantis.com/mke/3.5/ops/deploy-apps-k8s/pod-security-policies/psp-examples.html + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: seccomppolicy + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' +spec: + allowPrivilegeEscalation: false + allowedHostPaths: + - pathPrefix: /dev/null + readOnly: true + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' diff --git a/exercises/psp/test_psa.log b/exercises/psp/test_psa.log new file mode 100644 index 0000000..97b9565 --- /dev/null +++ b/exercises/psp/test_psa.log @@ -0,0 +1,87 @@ + +Checking namespace: namespace/cert-manager + Restricted: warnings +Warning: existing pods in namespace "cert-manager" violate the new PodSecurity enforce level "restricted:latest" +Warning: cert-manager-677874db78-p6t4d (and 2 other pods): unrestricted capabilities, seccompProfile + Baseline: ok + Recommended level: baseline +------------------------------------------------------------------- +Checking namespace: namespace/default + Restricted: ok + Baseline: ok + Recommended level: restricted +------------------------------------------------------------------- +Checking namespace: namespace/falco + Restricted: warnings +Warning: existing pods in namespace "falco" violate the new PodSecurity enforce level "restricted:latest" +Warning: falco-tq7z2: hostPath volumes, privileged, allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types, runAsNonRoot != true, seccompProfile + Baseline: warnings +Warning: existing pods in namespace "falco" violate the new PodSecurity enforce level "baseline:latest" +Warning: falco-tq7z2: hostPath volumes, privileged + Recommended level: privileged +------------------------------------------------------------------- +Checking namespace: namespace/gatekeeper + Restricted: warnings +Warning: existing pods in namespace "gatekeeper" violate the new PodSecurity enforce level "restricted:latest" +Warning: gatekeeper-audit-f7bd7bc6b-cnfrh (and 3 other pods): unrestricted capabilities + Baseline: ok + Recommended level: baseline +------------------------------------------------------------------- +Checking namespace: namespace/helm-exporter + Restricted: warnings +Warning: existing pods in namespace "helm-exporter" violate the new PodSecurity enforce level "restricted:latest" +Warning: helm-exporter-7684576787-x6928: allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile + Baseline: ok + Recommended level: baseline +------------------------------------------------------------------- +Checking namespace: namespace/kube-node-lease + Restricted: ok + Baseline: ok + Recommended level: restricted +------------------------------------------------------------------- +Checking namespace: namespace/kube-public + Restricted: ok + Baseline: ok + Recommended level: restricted +------------------------------------------------------------------- +Checking namespace: namespace/kube-system + Restricted: warnings +Warning: existing pods in namespace "kube-system" violate the new PodSecurity enforce level "restricted:latest" +Warning: coredns-64897985d-5xzht: unrestricted capabilities, runAsNonRoot != true, seccompProfile +Warning: etcd-minikube (and 3 other pods): host namespaces, hostPath volumes, allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types, runAsNonRoot != true +Warning: kube-proxy-sshvg: host namespaces, hostPath volumes, privileged, allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types, runAsNonRoot != true, seccompProfile +Warning: metrics-server-6b76bd68b6-j2mvb: allowPrivilegeEscalation != false, unrestricted capabilities, seccompProfile +Warning: storage-provisioner: host namespaces, hostPath volumes, allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types, runAsNonRoot != true, seccompProfile + Baseline: warnings +Warning: existing pods in namespace "kube-system" violate the new PodSecurity enforce level "baseline:latest" +Warning: etcd-minikube (and 4 other pods): host namespaces, hostPath volumes +Warning: kube-proxy-sshvg: host namespaces, hostPath volumes, privileged + Recommended level: privileged +------------------------------------------------------------------- +Checking namespace: namespace/promstack + Restricted: warnings +Warning: existing pods in namespace "promstack" violate the new PodSecurity enforce level "restricted:latest" +Warning: alertmanager-promstack-kube-prometheus-alertmanager-0 (and 1 other pod): seccompProfile +Warning: promstack-grafana-fcb5d8b98-s2zgc (and 1 other pod): allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile +Warning: promstack-kube-prometheus-operator-f67f4bb66-k924x: unrestricted capabilities, seccompProfile +Warning: promstack-prometheus-node-exporter-q79bz: host namespaces, hostPath volumes, hostPort, allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types, seccompProfile + Baseline: warnings +Warning: existing pods in namespace "promstack" violate the new PodSecurity enforce level "baseline:latest" +Warning: promstack-prometheus-node-exporter-q79bz: host namespaces, hostPath volumes, hostPort + Recommended level: privileged +------------------------------------------------------------------- +Checking namespace: namespace/robusta + Restricted: warnings +Warning: existing pods in namespace "robusta" violate the new PodSecurity enforce level "restricted:latest" +Warning: robusta-forwarder-6485d85ff6-ztk4s (and 1 other pod): unrestricted capabilities, runAsNonRoot != true, seccompProfile + Baseline: ok + Recommended level: baseline +------------------------------------------------------------------- +Checking namespace: namespace/vault + Restricted: warnings +Warning: existing pods in namespace "vault" violate the new PodSecurity enforce level "restricted:latest" +Warning: vault-0 (and 2 other pods): unrestricted capabilities, seccompProfile +Warning: vault-agent-injector-758ddd488-hh7cg: seccompProfile + Baseline: ok + Recommended level: baseline +------------------------------------------------------------------- diff --git a/exercises/rbac/makefile b/exercises/rbac/makefile new file mode 100644 index 0000000..f996118 --- /dev/null +++ b/exercises/rbac/makefile @@ -0,0 +1,37 @@ +# RBAC testing + +rbac: create-user deploy-roles deploy-role-scale + kubectl apply -f rbac/scale-deployment.yaml + @$(MAKE) -s switch-user1 + +create-user: + @mkdir -p rbac && cd rbac && \ + openssl genrsa -out user1.key 2048 && \ + openssl req -new -key user1.key -out user1.csr -subj '/CN=user1/O=group1' && \ + openssl x509 -req -in user1.csr -CA ~/.minikube/ca.crt -CAkey ~/.minikube/ca.key -CAcreateserial -out user1.crt -days 500 && \ + cp user1.key ~/.minikube/user1.key && \ + cp user1.csr ~/.minikube/user1.csr && \ + kubectl config set-credentials user1 --client-certificate=user1.crt --client-key=user1.key && \ + kubectl config set-context user1-minikube --namespace=tennant --user=user1 + +deploy-roles: + kubectl apply -f rbac/role.yaml + kubectl apply -f rbac/rolebinding.yaml + +deploy-role-scale: + kubectl apply -f rbac/role-scale.yaml --user=minikube + kubectl apply -f rbac/rolebinding-scale.yaml --user=minikube + +scale-up: + kubectl scale --replicas=3 deployment/scale-deployment + +scale-down: + kubectl scale --replicas=1 deployment/scale-deployment + +can-i-user1: + kubectl auth can-i --list --as=user1 + +switch-user1: + kubectl config set-context user1-minikube --namespace=tennant --user=user1 + kubectl config use-context user1-minikube + diff --git a/makefile b/makefile new file mode 100644 index 0000000..f43175d --- /dev/null +++ b/makefile @@ -0,0 +1,113 @@ + +CLUSTER_TYPE="minikube" + +.PHONY: separator + +# ------------------------------------------------- + +# Create a cluster, install and configure software +all: start helmfile_lint helmfile_sync enable_audit_log + +# Create a cluster +start: + cd cluster/${CLUSTER_TYPE} && make start + @@echo "-------------------------------------------------" + +# Delete a cluster +delete: + cd cluster/${CLUSTER_TYPE} && make delete + +# Follow audit log +audit_log: + cd cluster/${CLUSTER_TYPE} && make audit_log + +# Get kubeconfig for kubectl +kubeconfig: + cd cluster/${CLUSTER_TYPE} && make kubeconfig + +# ------------------------------------------------- + +# Sync a set of charts: Prometheus, Grafana, Alertmanager, Gatekeeper, Vault, cert-manager +helmfile_sync: + cd charts && helmfile sync + @@echo "-------------------------------------------------" + +# Lint charts to be deployed +helmfile_lint: + cd charts && helmfile lint + @@echo "-------------------------------------------------" + +# Enable audit log +enable_audit_log: + cd cluster/${CLUSTER_TYPE} && make enable_audit_log + @@echo "-------------------------------------------------" + +# Deprecated +prepare_secrets: + @@sed "s/replace_api_key/$$opsgenie_api_key/g" ./charts/prometheus/am-opsgenie.sample.yaml \ + | sed "s/replace_team_id/$$opsgenie_responder_id/g" > ./charts/prometheus/am-opsgenie.yaml + @@echo "./charts/prometheus/am-opsgenie.yaml generated" + +# ------------------------------------------------- + +# Install Google Microservices Demo +install_demo: + @$(MAKE) -s create_namespace NAMESPACE=tennant + kubectl apply -n tennant -f https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/main/release/kubernetes-manifests.yaml + +# Delete Google Microservices Demo +delete_demo: + kubectl delete -n tennant -f https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/main/release/kubernetes-manifests.yaml + +# Install Graylog +install_graylog: + helm repo add kongz https://charts.kong-z.com + helm upgrade --install --namespace "graylog" --create-namespace \ + graylog kongz/graylog \ + --set graylog.replicas=1 \ + --set tags.install-mongodb=true \ + --set tags.install-elasticsearch=true \ + --set graylog.elasticsearch.version=7 + +# Delete Graylog +proxy_graylog: + @echo "http://localhost:3010" + @echo "User: admin" + @PASS=$$(kubectl get secret --namespace graylog graylog -o "jsonpath={.data['graylog-password-secret']}" | base64 --decode) ; \ + echo "Password: $$PASS" + kubectl -n graylog port-forward svc/graylog-web 3010:9000 + +# Install ArgoCD +install_argocd: + kubectl create namespace argocd + kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml + +# kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/core-install.yaml + +# Delete ArgoCD +proxy_argocd: + @echo "http://localhost:8090" + @echo "user: admin" + @echo "pass:" + @kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo + kubectl port-forward svc/argocd-server -n argocd 8090:443 + +# ------------------------------------------------- + +proxy_prometheus: + # We don't use port 9090 to avoid collision with local installation of Prometheus + @echo "Navigate to http://localhost:9080" + kubectl -n promstack port-forward svc/promstack-kube-prometheus-prometheus 9080:9090 + +proxy_grafana: + @echo "Navigate to http://localhost:3000" + @echo "Default user: admin , password: prom-operator" + kubectl -n promstack port-forward svc/promstack-grafana 3000:80 + +proxy_alertmanager: + @echo "Navigate to http://localhost:9093" + kubectl -n promstack port-forward svc/promstack-kube-prometheus-alertmanager 9093:9093 + +proxy_robusta: + @echo "No need to proxy, visit:" + @echo "https://platform.robusta.dev/" diff --git a/sample.envrc b/sample.envrc new file mode 100644 index 0000000..54f9334 --- /dev/null +++ b/sample.envrc @@ -0,0 +1,15 @@ +export opsgenie_api_key=XXXXXXXXX +export opsgenie_responder_id=XXXXXXXXX + +export mailtrap_username=XXXXXXXXX +export mailtrap_password=XXXXXXXXX + +export robusta_signing_key="XXXXXXXXX" +export robusta_account_id="XXXXXXXXX" +export robusta_sink_ui="XXXXXXXXXXXXXXXXXXXXXXXXXXX" +export robusta_private_key='XXXXXXXXX +XXXXXXXXX +XXXXXXXXX' +export robusta_public_key='XXXXXXXXX +XXXXXXXXX +XXXXXXXXX' \ No newline at end of file