Tests for encryption with Kourier local gateway (knative#13263)

* Generate Secrets * Commit generated cert-secret.yaml * httpproxy enables tls client * httpproxy uses https when CA_CERT specified * Pass CA_CERT and SERVER_NAME env variables properly to tests * Avoid using cluster-local certificates for external services
davidggz · Sep 5, 2022 · 6d3d676 · 6d3d676
1 parent 3e07846
commit 6d3d676
Show file tree

Hide file tree

Showing 6 changed files with 187 additions and 8 deletions.
diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml
@@ -312,6 +312,8 @@ jobs:
         echo "SYSTEM_NAMESPACE=$SYSTEM_NAMESPACE" >> $GITHUB_ENV
         echo "GATEWAY_OVERRIDE=$GATEWAY_OVERRIDE" >> $GITHUB_ENV
         echo "GATEWAY_NAMESPACE_OVERRIDE=$GATEWAY_NAMESPACE_OVERRIDE" >> $GITHUB_ENV
+        echo "CA_CERT=$CA_CERT" >> $GITHUB_ENV
+        echo "SERVER_NAME=$SERVER_NAME" >> $GITHUB_ENV
 
     - name: Test ${{ matrix.test-suite }}
       run: |

diff --git a/test/config/tls/cert-secret.yaml b/test/config/tls/cert-secret.yaml
@@ -0,0 +1,30 @@
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ca-cert
+  namespace: serving-tests
+data:
+  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVQ2pESkw2RStjQ3FVVGdOdXNkY0g2Z2pNRStNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRWFNQmdHQTFVRUNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxdwpiR1V1WTI5dE1CNFhEVEl5TURnek1UQTVNVE14TVZvWERUSXpNRGd6TVRBNU1UTXhNVm93TWpFYU1CZ0dBMVVFCkNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxd2JHVXVZMjl0TUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBbkdWdVROa3pkcm10dEdqOWRWWjBiYlp2d3UwaQpjd1I5MUZ1U3NyRmVYZFNuV212MWJxTWZ6SGJWVkFoYTRRV2xwWktqc2M4bG9OalV6WEdLZ1o3Ulp0ajA0endKCnVubHJmSm5obTZOL2ZnYVNKem1RaEpjek9ZZ3VLM290dzN6dGxtZ3RpWE56VTBoUDc0bWNBMjlPS0NwbDdTNkkKWmwrV2xNUTdIWldqemY0ZUR3TmRqMDNkWm0zSGVweHVyWUFCMHZmMEk0L25ETHpxL0Jha2drVkpTTmdCWDFBLwo4TzluN0dCeGQ2NEU2WWxmY294UmZybHdNNkg0L2RmeTRMdTZYdlAzU3RvNWhWNWZsb2RzU1lNZ2xJNFE0dDZ0CmdTSUpjSGpyRExXL0RpSUkxQmw0aHpHelFtY0N3ZWFXbzQ4YkFsaFhOK1NFWm5iMDlRcE1oMHZ5N1FJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVaTFF2aGZvWldPaUY4ZWRlTVA1UUxOejJsUFl3SHdZRFZSMGpCQmd3Rm9BVQpaTFF2aGZvWldPaUY4ZWRlTVA1UUxOejJsUFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQW16dUJHZUxJWGkyOTBhUXh6UU1pSjRhdWtJTUQzNXRFa0IxSUFTT2RiQWVRWEhGb3I3NFMKbENwMSt1Rm5nUjJuY1EydUFjSjM3VUZhVm04d2t2UkN2bXg5bWN4anpwQ1F1bXpsMU1QY3pHdWxybjEreHArVAoxYlk2QUg1RFo0VXkrQ0JvMUpEaE0xUFVzdlJMOTcyQUpPU2tYWEpYajRQcWdSdStBUlU1eDk4K2QvbUFIemhPCkNCdW5iQTZTZEhLYjVMOTdxbCtqSzNFb0I2VERLNmNvNW5LSStHcnBDcTRKcllUZ1R1NG1uTkp0WHJEdWNpWnEKbG5DTEdWTTAvaFJ2NGRGMWZ4eGdLVkRMUmVpaitaM1lGQk1YYlFmb3B0c1o3RC9Tay85UnVVbkgzNUFtQnZRMwpNL21pRERaSTZDTVNtREQ5TCtsNmhMb2N3Zi9rS1BOVkJRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-certs
+  namespace: knative-serving
+data:
+  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRGhaYk8rUTJtbVhHb2QKeHZTMDJncW9LVHoxWWRTRVJhQzgwbElxMzB3NHNlblVXRFJva1J5dDBSOGU5R29Qd2hneGxnWGZoM296ODZYZQp4WncrclZJd1RZQVE3UHp0TnFFU04vQXRMQ1dBWHV0OTRTcSt2Z1JyTVY0dGFNUHBBV3VmaDB0WFhxeFR3TTJBCk43dE5yUi9oaXVrdkoxSTl2empDbGVJK2RXeEtWWWpNWFRWMXRwMEpxSThZVlNKOXp2TGxSODhzRCtqSFk0am0KN3JERnlTREVVQVd1Vlp1NVdJdEpqY1N6NnQycXlReVdMcDNWcC9GRU9rbGdQQTRkelg5ZlRETjlaMzdTR2JSaQppaXRNMEYxdTNSYUdJeGFhS0NFYXg3UW1zODJ1YXdwNyt0cWUwQlpkYXlQS01UUUJJM3VhWXVIMjRTZWQ3dkVnCjRaYUI0bnFGQWdNQkFBRUNnZ0VBS0RPRWlKWHJmUkdVbDdVSnBrd1JoSWErYWFIR1RzVkFjdzBzUEp0Uk0vZC8KbGpFWlArRko5VEtNVTNBU2pyYjJxN2x2V0x3SUxzWHhPcmVTTTVla1JoczhrVWhEb0dlUytQWGpMNXRsSU8xTgpJVW1NM3pKekJVOXIxYnVPM2JzMEgrTDRyQitscXRhRGtLL2dCMjJ1ZHdMWXJtRmNDTWxYYlZWZ1lmVjltQkF0CnFRZ0NOdzlRVGhtVEpCY0VwWUtnNVdoem9pdjNUQmdWWXRmQ1I0d0VSOGJFaVA0ckhTQ001TFpqZ1VwQ1ZzVUMKRkxzY2hOVDl1SS9EcDhXN2tGMGJ0TjFIcjNuZWZ4MzN4cERxRmxvSzloeXM0ODhnUit1cXUyZGZMN1NMZ1JPZApCaTVFVytwRWZaQzlZckNlcGI1QnBabEM5NkF4US9kUGhQMER2b3lBYlFLQmdRRDhKazVzVlIvc3k2YjMyWE5jCld6VnpmclZNaFFYa1ZzMDh5UmNTWmpxeWYvTjhqczVRMjR3SzcvLzlhYjVnRWtVcGgxZUVmQU5TaEJEVFQrQlkKK3QzVDZUSHNpMVArdEc1NFJEcFVIU1EvSW53bmoxcSt5Q3hsNjM5VE1OVUU2bFRjWkJMRW9ZUmExcklsdXg2VwpVaHZZdVpKN3E4dk5HZk9yT2Z0MWQyZkQzd0tCZ1FEazF0RUZCbHlUeUtuazZlY2ZvZmdxbnhHd0s3N2pDaUJPCmRpcGhMT1NkVzh0ZWt5d0J6R1ViU3d5YUhYcXB4UEhQblp1eXZ0dWxJak9waW82VGVQMzl6emZFQVZuVEw3b2IKb0t4S3dHbXFBRUpMUkF4NWtZMUsybSt0a1hYYUwzeDZJMCtESGlmSXdoR3hPSk1xd3ZZM2FkbW11bkkzUmdMYwpydTRzTFJsdUd3S0JnQ0VHdzI3ZEYzbGtrMUlUWVZEUGdZakhKK2dGNUdlc0Z1WEhVUVpQN1pCRHdoaW1lOCtMCmNpUmNteU1PSHFsbXV6aGRTZEZJalFiWjFYcFlGQUtUbVVxUVdNR3EzaTJXWklITUxmZW1lWURyZTJlVEYwZTEKNEZyWkphdzMwUzc3b25IYmlibkhqaFozMkkyb25MRUR3REg2M0h6bVc2TlpxdGphbDEwamJxdnhBb0dCQU1ZbQpiSjlaUHRpSXJQUVd4WmJTZlQwS3VCcEFCdTQ1V25nV1FlUWJKMnBLamZLNnBTUjVoQ0w1L1ZPRnF5MU41OFRLCnlJTWlXTGJJd3N0UHV0MWZxeThYTzBaeGxRSVZGYVhPbnVHcmN0Tk5uaG5tTnBjZHZhYlBObHlvMDgrMXhxZEwKNUJHNUR1SDdpYTVYT3JlUVVmcnhvUkdKNkZTTVB2WXdVdlBWcVd0NUFvR0FGSmx2MEFEaTY5eXpVaUU5Y2tZOQplYTJzVEFlZmJSbHNsaVBNdXNuWjdPSE81UzAzOURJZ01xSk9VOHlXVlRYR3FmTFhsZDJoZG9GeDRCSURwY1F3CkJGeHJ3d0s4eGVwY21zd29BNUk5Qlg0WTBMclQzeU5JbFdod0NJaE9RdlZnOXR3ZDdKOGRlY21IbVpFaXVRbTEKVUFpRHo5UHhlL0MwdC9sb2JGWHpDYTg9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lVUXFEUTkrWWRPdDUydWFaT0JMYStTNWdGWnJ3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRWFNQmdHQTFVRUNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxdwpiR1V1WTI5dE1CNFhEVEl5TURnek1UQTVNVE14TVZvWERUSXpNRGd6TVRBNU1UTXhNVm93TWpFVU1CSUdBMVVFCkF3d0xaWGhoYlhCc1pTNWpiMjB4R2pBWUJnTlZCQW9NRVV0dVlYUnBkbVVnUTI5dGJYVnVhWFI1TUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBNFdXenZrTnBwbHhxSGNiMHROb0txQ2s4OVdIVQpoRVdndk5KU0t0OU1PTEhwMUZnMGFKRWNyZEVmSHZScUQ4SVlNWllGMzRkNk0vT2wzc1djUHExU01FMkFFT3o4CjdUYWhFamZ3TFN3bGdGN3JmZUVxdnI0RWF6RmVMV2pENlFGcm40ZExWMTZzVThETmdEZTdUYTBmNFlycEx5ZFMKUGI4NHdwWGlQblZzU2xXSXpGMDFkYmFkQ2FpUEdGVWlmYzd5NVVmUExBL294Mk9JNXU2d3hja2d4RkFGcmxXYgp1VmlMU1kzRXMrcmRxc2tNbGk2ZDFhZnhSRHBKWUR3T0hjMS9YMHd6ZldkKzBobTBZb29yVE5CZGJ0MFdoaU1XCm1pZ2hHc2UwSnJQTnJtc0tlL3JhbnRBV1hXc2p5akUwQVNON21tTGg5dUVubmU3eElPR1dnZUo2aFFJREFRQUIKb3hvd0dEQVdCZ05WSFJFRUR6QU5nZ3RyYm1GMGFYWmxMbVJsZGpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQpHN2xraFI5VC9OMFIxLzc1WGJ5dWdlOWgrYXFHTzdxblAyTHg2aHo3bHdjOFM2VTVNS09EaW9zRXhMOStKYzhSCkorbTJjTmFsR3pOTG1ER3FDbDBJU3g4SGRhc1ZQbW9McFlpamFlRWNXS1JoVzZaTUNyVlorOEpnQWVmcTFSaFQKMnBqM05OTllWZ2NaWVJsV2NydlBnWlFZZ3JYTGhCWCtPKzRQWmNXUlZzem1yYncwWms1aE53ZWNWc1lvbHRLVwpCdEpCNmx5WDBLcDdNcURqUXBnZUR4WnVTbElMaktTK1J6ejU0L3ZxTDhpQTJrN0Z6TWNUS1IydXhTRGtwK2lZCkQrQ2cyenBnb3c2RFJxeEowWmJ3YXZRY3ozWHQrc0FUYjZIcG10WFNyVHk1R2E4L0R2U1ZvSGl2YlNTMkJ5ZloKQ3BBZkhLT0FBUGdHbUFOaGJpQ2VhZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
diff --git a/test/config/tls/generate.sh b/test/config/tls/generate.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script generates test/config/tls/cert-secret.yaml.
+
+san="knative.dev"
+
+# Create CA key and cert
+openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=Knative Community/CN=example.com' -keyout rootCAKey.pem -out rootCACert.pem
+
+# Create server key
+openssl req -out tls.csr -newkey rsa:2048 -nodes -keyout tls.key -subj "/CN=example.com/O=Knative Community" -addext "subjectAltName = DNS:$san"
+
+# Create server certs
+openssl x509 -req -extfile <(printf "subjectAltName=DNS:$san") -days 365 -in tls.csr -CA rootCACert.pem -CAkey rootCAKey.pem -CAcreateserial -out tls.crt
+
+CA_CERT=$(cat rootCACert.pem | base64  | tr -d '\n')
+TLS_KEY=$(cat tls.key | base64  | tr -d '\n')
+TLS_CERT=$(cat tls.crt | base64  | tr -d '\n')
+
+cat <<EOF > cert-secret.yaml
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ca-cert
+  namespace: serving-tests
+data:
+  ca.crt: ${CA_CERT}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-certs
+  namespace: knative-serving
+data:
+  tls.key: ${TLS_KEY}
+  tls.crt: ${TLS_CERT}
+EOF
+
+# Clean up
+rm -f rootCACert.pem rootCAKey.pem tls.key tls.crt tls.csr rootCACert.srl
diff --git a/test/e2e-common.sh b/test/e2e-common.sh
@@ -307,6 +307,10 @@ function install() {
     YTT_FILES+=("${REPO_ROOT_DIR}/test/config/resource-quota/resource-quota.yaml")
   fi
 
+  if (( ENABLE_TLS )); then
+    YTT_FILES+=("${REPO_ROOT_DIR}/test/config/tls/cert-secret.yaml")
+  fi
+
   local ytt_result=$(mktemp)
   local ytt_post_install_result=$(mktemp)
   local ytt_flags=""
@@ -360,11 +364,15 @@ function install() {
 
   if (( ENABLE_TLS )); then
     echo "Patch to config-network to enable internal encryption"
-    kubectl patch configmap/config-network \
-      -n ${SYSTEM_NAMESPACE} \
-      --type merge \
-      -p '{"data":{"internal-encryption":"true"}}'
-
+    toggle_feature internal-encryption true config-network
+    if [[ "$INGRESS_CLASS" == "kourier.ingress.networking.knative.dev" ]]; then
+      echo "Point Kourier local gateway to custom server certificates"
+      toggle_feature cluster-cert-secret server-certs config-kourier
+      # This needs to match the name of Secret in test/config/tls/cert-secret.yaml
+      export CA_CERT=ca-cert
+      # This needs to match $san from test/config/tls/generate.sh
+      export SERVER_NAME=knative.dev
+    fi
     echo "Restart activator to mount the certificates"
     kubectl delete pod -n ${SYSTEM_NAMESPACE} -l app=activator
     kubectl wait --timeout=60s --for=condition=Available deployment  -n ${SYSTEM_NAMESPACE} activator

diff --git a/test/e2e/service_to_service_test.go b/test/e2e/service_to_service_test.go
@@ -24,12 +24,14 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
 	netapi "knative.dev/networking/pkg/apis/networking"
+	ptr "knative.dev/pkg/ptr"
 	pkgTest "knative.dev/pkg/test"
 	ingress "knative.dev/pkg/test/ingress"
 	"knative.dev/pkg/test/logstream"
@@ -45,6 +47,8 @@ const (
 	targetHostEnv      = "TARGET_HOST"
 	gatewayHostEnv     = "GATEWAY_HOST"
 	helloworldResponse = "Hello World! How about some tasty noodles?"
+	caCertDirectory    = "/var/lib/knative/ca"
+	caCertPath         = caCertDirectory + "/ca.crt"
 )
 
 // testCases for table-driven testing.
@@ -109,6 +113,16 @@ func testProxyToHelloworld(t *testing.T, clients *test.Clients, helloworldURL *u
 		})
 	}
 
+	caSecretName := os.Getenv("CA_CERT")
+
+	// External services use different TLS certificates than cluster-local services.
+	// Not passing CA_CERT will make the httpproxy use plain http to connect to the
+	// target service.
+	if caSecretName != "" && !accessibleExternal {
+		envVars = append(envVars, []corev1.EnvVar{{Name: "CA_CERT", Value: caCertPath},
+			{Name: "SERVER_NAME", Value: os.Getenv("SERVER_NAME")}}...)
+	}
+
 	// Set up httpproxy app.
 	t.Log("Creating a Service for the httpproxy test app.")
 	names := test.ResourceNames{
@@ -118,11 +132,24 @@ func testProxyToHelloworld(t *testing.T, clients *test.Clients, helloworldURL *u
 
 	test.EnsureTearDown(t, clients, &names)
 
-	resources, err := v1test.CreateServiceReady(t, clients, &names,
+	serviceOptions := []rtesting.ServiceOption{
 		rtesting.WithEnv(envVars...),
 		rtesting.WithConfigAnnotations(map[string]string{
 			"sidecar.istio.io/inject": strconv.FormatBool(inject),
-		}))
+		}),
+	}
+
+	if caSecretName != "" && !accessibleExternal {
+		serviceOptions = append(serviceOptions, rtesting.WithVolume("ca-certs", caCertDirectory, corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: caSecretName,
+				Optional:   ptr.Bool(false),
+			}}),
+		)
+	}
+
+	resources, err := v1test.CreateServiceReady(t, clients, &names, serviceOptions...)
+
 	if err != nil {
 		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
 	}

diff --git a/test/test_images/httpproxy/httpproxy.go b/test/test_images/httpproxy/httpproxy.go
@@ -17,6 +17,9 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"flag"
 	"log"
 	"os"
@@ -61,12 +64,17 @@ func main() {
 		routedHost = gateway
 	}
 
+	scheme := "http"
+	if caCert := os.Getenv("CA_CERT"); caCert != "" {
+		scheme = "https"
+	}
 	target := &url.URL{
-		Scheme: "http",
+		Scheme: scheme,
 		Host:   routedHost,
 	}
 	log.Print("target is ", target)
 	proxy := httputil.NewSingleHostReverseProxy(target)
+	proxy.Transport = newTLSEnabledTransport()
 	proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) {
 		log.Print("error reverse proxying request: ", err)
 		http.Error(w, err.Error(), http.StatusBadGateway)
@@ -84,3 +92,39 @@ func main() {
 	log.Print("Listening on address: ", address)
 	test.ListenAndServeGracefully(address, handler)
 }
+
+func newTLSEnabledTransport() http.RoundTripper {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if caCert := os.Getenv("CA_CERT"); caCert != "" {
+		rootCAs, err := createRootCAs(caCert)
+		if err != nil {
+			log.Fatal(err)
+			return transport
+		}
+		transport.TLSClientConfig = &tls.Config{
+			RootCAs: rootCAs,
+			// If SERVER_NAME is not set the empty value will make the
+			// TLS client infer the ServerName from the hostname.
+			ServerName: os.Getenv("SERVER_NAME"),
+		}
+	}
+	return transport
+}
+
+func createRootCAs(caCertFile string) (*x509.CertPool, error) {
+	pemData, err := os.ReadFile(caCertFile)
+	if err != nil {
+		return nil, err
+	}
+	rootCAs, err := x509.SystemCertPool()
+	if rootCAs == nil || err != nil {
+		if err != nil {
+			log.Printf("Failed to load cert poll from system: %v. Will create a new cert pool.", err)
+		}
+		rootCAs = x509.NewCertPool()
+	}
+	if !rootCAs.AppendCertsFromPEM(pemData) {
+		return nil, errors.New("failed to add the certificate to the root CA")
+	}
+	return rootCAs, nil
+}