Implement node synthetic ref (#108)

Implement node synthetic refs
kubemod · Dec 23, 2022 · 95c43bc · 95c43bc
1 parent 565574d
commit 95c43bc
Show file tree

Hide file tree

Showing 23 changed files with 1,037 additions and 35 deletions.
diff --git a/Makefile b/Makefile
@@ -82,7 +82,8 @@ vet:
 
 # Generate code
 .PHONY: generate
-generate: controller-gen wire
+generate: controller-gen wire mockgen
+	$(MOCKGEN) -destination ./mocks/k8s_client_mock.go -package mocks sigs.k8s.io/controller-runtime/pkg/client Client
 	$(CONTROLLER_GEN) object:headerFile="misc/boilerplate.go.txt" paths="./..."
 	$(WIRE) ./...
 
@@ -138,3 +139,19 @@ else
 WIRE=$(shell which wire)
 endif
 
+# find or download mockgen
+.PHONY: mockgen
+mockgen:
+ifeq (, $(shell which mockgen))
+	@{ \
+	set -e ;\
+	MOCKGEN_TMP_DIR=$$(mktemp -d) ;\
+	cd $$MOCKGEN_TMP_DIR ;\
+	go mod init tmp ;\
+	go install github.com/golang/mock/mockgen@v1.6.0 ;\
+	rm -rf $$MOCKGEN_TMP_DIR ;\
+	}
+MOCKGEN=$(GOBIN)/mockgen
+else
+MOCKGEN=$(shell which mockgen)
+endif
diff --git a/README.md b/README.md
@@ -858,13 +858,17 @@ By default, the following namespaces are tagged with the above label:
 
 ### Synthetic references
 
-KubeMod 0.17.0 introduces `syntheticRefs` - a map of external resource manifests injected at the root of every Kubernetes resource processed by KubeMod.
+KubeMod 0.17.0 introduced `syntheticRefs` - a map of external resource manifests injected at the root of every Kubernetes resource processed by KubeMod.
 
-Currently the only external manifest injected in `syntheticRefs` is the manifest of the `namespace` of namespaced objects.
-This unlocks use cases where a ModRule can be matched against objects not only based on their own manifest, but also the manifests of their namespaces.
+Synthetic references unlock use cases where a ModRule can be matched against objects not only based on their own manifest, but also the manifests of their namespaces.
 
 In addition, since `syntheticRefs` exists in the body of the target resource, it can be used when constructing `patch` values.
 
+Currently KubeMod injects the following manifests in `syntheticRefs`:
+
+- `namespace`: The manifest of the namespace of the target, if the target is a namespaced object.
+- `node`: The manifest of the node of a pod (See [Node synthetic references](#node-synthetic-references) below for more information).
+
 Here's an example ModRule which matches all pods created in namespaces labeled with `color` equal to `blue`.
 The ModRule mutates those pods by tagging them with a label `flavor`, whose value is inherited from the `flavor` label of the pod's namespace.
 
@@ -918,6 +922,57 @@ The `syntheticRefs` map exists in the object's manifest only for the purpose of
 
 It is not actually inserted in the resulting resource manifest ultimately sent to the cluster.
 
+#### Node synthetic references
+
+KubeMod 0.19.0 introduced `node` synthetic reference for ModRules targeting pod manifests.
+
+In order to capture the node which a pod has been scheduled on, KubeMod listens to pod scheduling events.
+
+If KubeMod intercepts a pod scheduling event for a pod which has annotation `ref.kubemod.io/inject-node-ref` set to `"true"`, KubeMod updates the pod by injecting annotation `ref.kubemod.io/node` whose value is set to the name of the node.
+
+This triggers an `UPDATE` operation, which is again captured by KubeMod. When KubeMod intercepts pod operations for pods with annotation `ref.kubemod.io/node`, it injects the node manifest into the pod's synthetic references, thus making them available for ModRule matching and patching.
+
+This enables a wide array of use cases not natively supported by Kubernetes (see https://github.com/kubernetes/kubernetes/issues/40610).
+
+For example, the following cluster-wide ModRule will inject a pod with it's node's availability region and zone, as soon as the pod gets scheduled to a node:
+
+```yaml
+apiVersion: api.kubemod.io/v1beta1
+kind: ModRule
+metadata:
+  name: inject-node-annotations
+  namespace: kubemod-system
+spec:
+  type: Patch
+  targetNamespaceRegex: ".*"
+  admissionOperations:
+    - UPDATE
+
+  match:
+    # Match pods...
+    - select: '$.kind'
+      matchValue: 'Pod'
+    # ...which have access to the node's manifest through the synthetic ref injected by KubeMod.
+    - select: '$.syntheticRefs.node.metadata.labels'
+
+  patch:
+    # Grab the node's region and zone and put them in the pod's corresponding labels.
+    - op: add
+      path: /metadata/labels/topology.kubernetes.io~1region
+      value: '"{{ index .Target.syntheticRefs.node.metadata.labels "topology.kubernetes.io/region"}}"'
+    - op: add
+      path: /metadata/labels/topology.kubernetes.io~1zone
+      value: '"{{ index .Target.syntheticRefs.node.metadata.labels "topology.kubernetes.io/zone"}}"'
+```
+
+The above ModRule will apply to any pod created in any namespace as long as it has the following annotation:
+
+```yaml
+ref.kubemod.io/inject-node-ref: "true"
+```
+
+If you want to have this rule apply to pods which don't have this annotation, you can create another ModRule which injects `ref.kubemod.io/inject-node-ref` into any pod that matches a given criteria, or to all pods that are created in the cluster.
+
 ### Target resources
 
 By default, KubeMod targets the following list of resources:

diff --git a/app/operatorapp.go b/app/operatorapp.go
@@ -36,6 +36,7 @@ func NewKubeModOperatorApp(
 	manager manager.Manager,
 	modRuleReconciler *controllers.ModRuleReconciler,
 	coreDragnetWebhookHandler *core.DragnetWebhookHandler,
+	corePodBindingWebhookHandler *core.PodBindingWebhookHandler,
 	log logr.Logger,
 ) (*KubeModOperatorApp, error) {
 
@@ -66,6 +67,13 @@ func NewKubeModOperatorApp(
 		},
 	)
 
+	hookServer.Register(
+		"/podbinding-webhook",
+		&webhook.Admission{
+			Handler: corePodBindingWebhookHandler,
+		},
+	)
+
 	// Start the manager.
 	setupLog.Info("starting manager")
 	if err := manager.Start(ctrl.SetupSignalHandler()); err != nil {

diff --git a/app/wire.go b/app/wire.go
@@ -43,6 +43,7 @@ func InitializeKubeModOperatorApp(
 		core.NewModRuleStoreItemFactory,
 		core.NewModRuleStore,
 		core.NewDragnetWebhookHandler,
+		core.NewPodBindingWebhookHandler,
 		controllers.NewModRuleReconciler,
 		NewControllerManager,
 		NewKubeModOperatorApp,

diff --git a/app/wire_gen.go b/app/wire_gen.go
diff --git a/config/default/cluster_role_patch.yaml b/config/default/cluster_role_patch.yaml
@@ -5,7 +5,20 @@
     - ""
     resources:
     - namespaces
+    - nodes
     verbs:
     - list
     - get
     - watch
+- op: add
+  path: /rules/-1
+  value:
+    apiGroups:
+    - ""
+    resources:
+    - pods
+    verbs:
+    - list
+    - get
+    - watch
+    - update
diff --git a/config/kubemod-crt/kustomization.yaml b/config/kubemod-crt/kustomization.yaml
@@ -4,4 +4,4 @@ resources:
 images:
 - name: kubemod-crt-image
   newName: kubemod/kubemod-crt
-  newTag: v1.2.1
+  newTag: v1.3.0
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: kubemod/kubemod
-  newTag: v0.18.1
+  newTag: latest
diff --git a/config/webhook/webhooks.yaml b/config/webhook/webhooks.yaml
@@ -77,7 +77,33 @@ webhooks:
     - UPDATE
     - DELETE
     scope: '*'
-
+- name: podbinding.kubemod.io
+  clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /podbinding-webhook
+  failurePolicy: Fail
+  sideEffects: None
+  timeoutSeconds: 10
+  admissionReviewVersions: ["v1beta1"]
+  namespaceSelector:
+    matchExpressions:
+      - key: admission.kubemod.io/ignore
+        operator: NotIn
+        values: ["true"]
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    resources:
+    - pods/binding
+    operations:
+    - CREATE
+    - UPDATE
+    scope: '*'
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

diff --git a/core/dragnetwebhook.go b/core/dragnetwebhook.go
@@ -18,9 +18,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/kubemod/kubemod/api/v1beta1"
 	"strings"
 
+	"github.com/kubemod/kubemod/api/v1beta1"
+
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -41,7 +42,7 @@ type DragnetWebhookHandler struct {
 func NewDragnetWebhookHandler(manager manager.Manager, modRuleStore *ModRuleStore, log logr.Logger) *DragnetWebhookHandler {
 	return &DragnetWebhookHandler{
 		client:       manager.GetClient(),
-		log:          log.WithName("modrule-webhook"),
+		log:          log.WithName("dragnet-webhook"),
 		modRuleStore: modRuleStore,
 	}
 }
@@ -72,7 +73,7 @@ func (h *DragnetWebhookHandler) Handle(ctx context.Context, req admission.Reques
 
 	if err != nil {
 		log.Error(err, "Failed to inject syntheticRefs into object manifest")
-		return admission.Allowed("failed to obtain namespace")
+		return admission.Allowed("failed to inject syntheticRefs into object manifest")
 	}
 
 	// First run patch operations.
@@ -107,38 +108,36 @@ func (h *DragnetWebhookHandler) Handle(ctx context.Context, req admission.Reques
 func (h *DragnetWebhookHandler) injectSyntheticRefs(ctx context.Context, originalJSON []byte, namespace string) ([]byte, error) {
 	obj := &unstructured.Unstructured{}
 	syntheticRefs := make(map[string]interface{})
+	var err error
 
 	// If the target is a namespaced object, grab the namespace from the manager's client.
 	if namespace != "" {
-		ns := &unstructured.Unstructured{}
-		ns.SetGroupVersionKind(schema.GroupVersionKind{
-			Group:   "",
-			Kind:    "Namespace",
-			Version: "v1",
-		})
-
-		err := h.client.Get(ctx, client.ObjectKey{Name: namespace}, ns)
-
+		err = h.injectNamespaceSyntheticRef(ctx, namespace, syntheticRefs)
 		if err != nil {
-			return nil, fmt.Errorf("failed to retrieve namespace '%s' : %v", namespace, err)
+			return nil, fmt.Errorf("failed to inject namespace synthetic ref: %v", err)
 		}
-
-		// Remove managedFields - we don't need this and it only litters the namespace manifest.
-		ns.SetManagedFields(nil)
-		// Remove annotation kubectl.kubernetes.io/last-applied-configuration as it simply duplicates the namespace manifest.
-		annotations := ns.GetAnnotations()
-		delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
-		ns.SetAnnotations(annotations)
-
-		syntheticRefs["namespace"] = ns
 	}
 
-	err := json.Unmarshal(originalJSON, obj)
+	err = json.Unmarshal(originalJSON, obj)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode webhook request object's manifest into JSON: %v", err)
 	}
 
+	// If the target object is a pod, check if the pod has a node name set by KubeMod in annotation ref.kubemod.io/nodename
+	// and if yes, inject the pod's node manifest into the synthetic refs.
+	if isPod(obj.UnstructuredContent()) {
+		nodeName, ok, err := unstructured.NestedString(obj.UnstructuredContent(), "metadata", "annotations", "ref.kubemod.io/nodename")
+
+		if ok && err == nil && nodeName != "" {
+			err = h.injectPodNodeSyntheticRef(ctx, nodeName, syntheticRefs)
+
+			if err != nil {
+				return nil, fmt.Errorf("failed to inject node synthetic ref: %v", err)
+			}
+		}
+	}
+
 	// Set KubeMod syntheticRefs field.
 	obj.UnstructuredContent()["syntheticRefs"] = syntheticRefs
 
@@ -148,3 +147,60 @@ func (h *DragnetWebhookHandler) injectSyntheticRefs(ctx context.Context, origina
 
 	return json.Marshal(obj)
 }
+
+func isPod(obj map[string]interface{}) bool {
+	kind, ok, err := unstructured.NestedString(obj, "kind")
+
+	if ok && err == nil && kind == "Pod" {
+		return true
+	}
+
+	return false
+}
+
+func (h *DragnetWebhookHandler) injectNamespaceSyntheticRef(ctx context.Context, namespace string, syntheticRefs map[string]interface{}) error {
+	ns := &unstructured.Unstructured{}
+	ns.SetGroupVersionKind(schema.GroupVersionKind{
+		Group:   "",
+		Kind:    "Namespace",
+		Version: "v1",
+	})
+
+	err := h.client.Get(ctx, client.ObjectKey{Name: namespace}, ns)
+
+	if err != nil {
+		return fmt.Errorf("failed to retrieve namespace '%s' : %v", namespace, err)
+	}
+
+	// Remove managedFields - we don't need this and it only litters the namespace manifest.
+	ns.SetManagedFields(nil)
+	// Remove annotation kubectl.kubernetes.io/last-applied-configuration as it simply duplicates the namespace manifest.
+	annotations := ns.GetAnnotations()
+	delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
+	ns.SetAnnotations(annotations)
+
+	syntheticRefs["namespace"] = ns
+	return nil
+}
+
+func (h *DragnetWebhookHandler) injectPodNodeSyntheticRef(ctx context.Context, nodeName string, syntheticRefs map[string]interface{}) error {
+	node := &unstructured.Unstructured{}
+
+	node.SetGroupVersionKind(schema.GroupVersionKind{
+		Group:   "",
+		Kind:    "Node",
+		Version: "v1",
+	})
+
+	err := h.client.Get(ctx, client.ObjectKey{Name: nodeName}, node)
+
+	if err != nil {
+		return fmt.Errorf("failed to retrieve node '%s': %v", nodeName, err)
+	}
+
+	// Remove managedFields - we don't need this and it only litters the manifest.
+	node.SetManagedFields(nil)
+
+	syntheticRefs["node"] = node
+	return nil
+}