add e2e test for cacert rotation (istio#47641)

* add e2e test for cacert rotation * lint & gen * add log * more fix * unmarshal * nit * address comment * update * wait a little longer
WillardHu · Nov 10, 2023 · e7c7c1a · e7c7c1a
1 parent b08d9d1
commit e7c7c1a
Show file tree

Hide file tree

Showing 5 changed files with 258 additions and 4 deletions.
diff --git a/pkg/test/framework/features/features.yaml b/pkg/test/framework/features/features.yaml
@@ -88,6 +88,7 @@ features:
       file-mounted-certs:
       ecc-signature-algorithm:
       multiple-root:
+      cacert-rotation:
     user:
     ingress:
       mtls:

diff --git a/samples/certs/README.md b/samples/certs/README.md
@@ -8,6 +8,7 @@ The included sample files are:
 
 - `root-cert.pem`: root CA certificate.
 - `root-cert-alt.pem`: alternative CA certificate.
+- `root-cert-combined.pem`: combine `root-cert.pem` and `root-cert-alt.pem` into a single file.
 - `ca-[cert|key].pem`: Citadel intermediate certificate and corresponding private key.
 - `ca-[cert-alt|key-alt].pem`: alternative intermediate certificate and corresponding private key.
 - `cert-chain.pem`: certificate trust chain.

diff --git a/samples/certs/root-cert-combined.pem b/samples/certs/root-cert-combined.pem
@@ -0,0 +1,54 @@
+-----BEGIN CERTIFICATE-----
+MIID7TCCAtWgAwIBAgIJAOIRDhOcxsx6MA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD
+VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJU3Vubnl2YWxl
+MQ4wDAYDVQQKDAVJc3RpbzENMAsGA1UECwwEVGVzdDEQMA4GA1UEAwwHUm9vdCBD
+QTEiMCAGCSqGSIb3DQEJARYTdGVzdHJvb3RjYUBpc3Rpby5pbzAgFw0xODAxMjQx
+OTE1NTFaGA8yMTE3MTIzMTE5MTU1MVowgYsxCzAJBgNVBAYTAlVTMRMwEQYDVQQI
+DApDYWxpZm9ybmlhMRIwEAYDVQQHDAlTdW5ueXZhbGUxDjAMBgNVBAoMBUlzdGlv
+MQ0wCwYDVQQLDARUZXN0MRAwDgYDVQQDDAdSb290IENBMSIwIAYJKoZIhvcNAQkB
+FhN0ZXN0cm9vdGNhQGlzdGlvLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA38uEfAatzQYqbaLou1nxJ348VyNzumYMmDDt5pbLYRrCo2pS3ki1ZVDN
+8yxIENJFkpKw9UctTGdbNGuGCiSDP7uqF6BiVn+XKAU/3pnPFBbTd0S33NqbDEQu
+IYraHSl/tSk5rARbC1DrQRdZ6nYD2KrapC4g0XbjY6Pu5l4y7KnFwSunnp9uqpZw
+uERv/BgumJ5QlSeSeCmhnDhLxooG8w5tC2yVr1yDpsOHGimP/mc8Cds4V0zfIhQv
+YzfIHphhE9DKjmnjBYLOdj4aycv44jHnOGc+wvA1Jqsl60t3wgms+zJTiWwABLdw
+zgMAa7yxLyoV0+PiVQud6k+8ZoIFcwIDAQABo1AwTjAdBgNVHQ4EFgQUOUYGtUyh
+euxO4lGe4Op1y8NVoagwHwYDVR0jBBgwFoAUOUYGtUyheuxO4lGe4Op1y8NVoagw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEANXLyfAs7J9rmBamGJvPZ
+ltx390WxzzLFQsBRAaH6rgeipBq3dR9qEjAwb6BTF+ROmtQzX+fjstCRrJxCto9W
+tC8KvXTdRfIjfCCZjhtIOBKqRxE4KJV/RBfv9xD5lyjtCPCQl3Ia6MSf42N+abAK
+WCdU6KCojA8WB9YhSCzza3aQbPTzd26OC/JblJpVgtus5f8ILzCsz+pbMimgTkhy
+AuhYRppJaQ24APijsEC9+GIaVKPg5IwWroiPoj+QXNpshuvqVQQXvGaRiq4zoSnx
+xAJz+w8tjrDWcf826VN14IL+/Cmqlg/rIfB5CHdwVIfWwpuGB66q/UiPegZMNs8a
+3g==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFFDCCAvygAwIBAgIUXl9kIYp2G+37cwt+ruWqmKHpFVkwDQYJKoZIhvcNAQEL
+BQAwIjEOMAwGA1UECgwFSXN0aW8xEDAOBgNVBAMMB1Jvb3QgQ0EwHhcNMjMwMjIw
+MDM1NjI2WhcNMzMwMjE3MDM1NjI2WjAiMQ4wDAYDVQQKDAVJc3RpbzEQMA4GA1UE
+AwwHUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMS9oq7l
+OR+vqj+99FcquCZiZB9d3AGmn4CUIzUsHLKwG/H4OZucHNaI+C/2IE+cHpqHq1RV
+XqOdE5fMoBsHTjRN24K/WMHVr76IiTdla3e5OGvb8XtFTqH80bPcahWU6J5SKaP5
+nuj6D0OCDuPgV5fDNkMBp6qH3b+zbSBLDKLyepMeHUdfUXKuUjAFCRzPuKuCzJy+
+xNHu61OuRzILSUL7O8kTSK/1iz0mIFAqSxeS6AFDGsQIJRhKEhQSbmH924dGGQ5p
+7bm8mFEYPYzyEw7l6zqaEKYEzoVhQrulJzUEVITVK9npW/GAREQK8KcabCakJf6L
+wiTMZkWzrY2h7d7U/4Ib/7N3/1HREG6rLjZy08owaf09PNhKE1eqc32rwcJUdbsq
+PiRqPRuIuGtNG69/CT+4I7liKoErJrxy4GfAxLRcFrkTA9Smo8lHsShbW/RvCEdr
+2eXHolxmfgogDr3kYkrG3jYgUyYqaNIdasVQwRkfQByQpxp3ItL66NLLd1jn9ImO
+IHAfWF2CCXdJtXKzksswOs/UW8gu8Y33YzfHZMxRQu5MlDEgKR6TJMDymYVedFAf
+BeqpJepIN4zhpV/DnfC4qfhKf5KWCPWHkZgOKwl6sxOVk25ozDeiatXMfHFztTJI
+1hWRF9cQNWdmHwVx19Yi2VfMwpgsnLDbeb+/AgMBAAGjQjBAMB0GA1UdDgQWBBQV
+XzlSBYPXo+qiYX/1gUkx+t9GrTAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQE
+AwIC5DANBgkqhkiG9w0BAQsFAAOCAgEAQj5EFGuY8jnXsvrcDEyo3kl+GAar60p3
+OB8BvMQzywYzmalztRluQr77dsTCDo9w+SL7/AKsJqljNJmzQg0Yst0YzPFAFUgt
+8PdCFDgjuSfzlrKukcRj7yd2rzs+MQP3amOuWIj5gq7lUWAPNMp0bhU1lIS2eAk7
+Ew8s4BkgMI2fW8z22uEw49j+720n9q3xIpmfGA7SISe32Z3bu10fXrsIii++Zo9p
+ItX7B5pKaLCwANjUejzC3NZ5HdL+yV9dXWXrVZrTvdFxRiqhNJtV64YTsSH+kVp8
+cCYyNA90CBVQU8ZyomNnOxqOsrBL1NvZllBmX1f3SnVa2Kw+RxJLtEx9lmKT3aiX
+v0kPPhoU+qpA3eOVvbT0CbSBEl56wclPMd7xYMykkNU/AVc3lZ7YFcjb19YNxzff
+AKktnJrFx3FHf03aeAnvJ35FxitrcpV4NvLvq5ViCHVw6IMO5h274Z/HoGHVvcle
+OPtLRiI5Fkaexa6Y/+SPMDFsCfQJzL4ZWmnGSA+Z6YMD5atviQGEbSP4bCwIgOax
+dsSyRo1cTRaOVCGVxed/s+ChbRnvljPd35zl79o+1zOOxJ2ttswYJRkjSsvn+BCL
+GZj8c/5MkLtTwvL62wZVhCIHtZoS4LNUqdkyYvRJMZTV72YuEPTfl8NKhmMislqR
+5LPKUi9adm4=
+-----END CERTIFICATE-----
diff --git a/tests/integration/security/cacert_rotation/main_test.go b/tests/integration/security/cacert_rotation/main_test.go
@@ -0,0 +1,192 @@
+//go:build integ
+// +build integ
+
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cacertrotation
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	admin "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+
+	"istio.io/istio/pkg/security"
+	"istio.io/istio/pkg/test/framework"
+	"istio.io/istio/pkg/test/framework/components/echo"
+	"istio.io/istio/pkg/test/framework/components/echo/check"
+	"istio.io/istio/pkg/test/framework/components/echo/common/deployment"
+	"istio.io/istio/pkg/test/framework/components/echo/echotest"
+	"istio.io/istio/pkg/test/framework/components/echo/match"
+	"istio.io/istio/pkg/test/framework/components/istio"
+	"istio.io/istio/pkg/test/framework/components/istioctl"
+	"istio.io/istio/pkg/test/framework/components/namespace"
+	"istio.io/istio/pkg/test/framework/label"
+	"istio.io/istio/pkg/test/framework/resource"
+	"istio.io/istio/pkg/test/util/retry"
+	"istio.io/istio/pkg/util/protomarshal"
+	"istio.io/istio/tests/integration/security/util/cert"
+	"istio.io/istio/tests/integration/security/util/reachability"
+)
+
+var apps deployment.SingleNamespaceView
+
+func TestMain(m *testing.M) {
+	framework.
+		NewSuite(m).
+		Label(label.CustomSetup).
+		Setup(istio.Setup(nil, setupConfig, cert.CreateCASecret)).
+		Setup(deployment.SetupSingleNamespace(&apps, deployment.Config{})).
+		Setup(func(ctx resource.Context) error {
+			return reachability.CreateCustomInstances(&apps)
+		}).
+		Run()
+}
+
+func setupConfig(_ resource.Context, cfg *istio.Config) {
+	if cfg == nil {
+		return
+	}
+	cfgYaml := `
+values:
+  pilot:
+    env:
+      ISTIO_MULTIROOT_MESH: true
+  meshConfig:
+    defaultConfig:
+      proxyMetadata:
+        PROXY_CONFIG_XDS_AGENT: "true"
+`
+	cfg.ControlPlaneValues = cfgYaml
+}
+
+func TestReachability(t *testing.T) {
+	framework.NewTest(t).
+		Features("security.peer.cacert-rotation").
+		Run(func(t framework.TestContext) {
+			istioCfg := istio.DefaultConfigOrFail(t, t)
+			istioCtl := istioctl.NewOrFail(t, t, istioctl.Config{})
+			namespace.ClaimOrFail(t, t, istioCfg.SystemNamespace)
+
+			from := apps.EchoNamespace.A
+			to := apps.EchoNamespace.B
+			fromAndTo := from.Append(to)
+
+			lastUpdateTime, err := getWorkloadCertLastUpdateTime(t, from[0], istioCtl)
+			if err != nil {
+				t.Errorf("failed to get workload cert last update time: %v", err)
+			}
+
+			// Verify traffic works between a and b
+			echotest.New(t, fromAndTo).
+				WithDefaultFilters(1, 1).
+				FromMatch(match.ServiceName(from.NamespacedName())).
+				ToMatch(match.ServiceName(to.NamespacedName())).
+				Run(func(t framework.TestContext, from echo.Instance, to echo.Target) {
+					// Verify mTLS works between a and b
+					opts := echo.CallOptions{
+						To: to,
+						Port: echo.Port{
+							Name: "http",
+						},
+					}
+					opts.Check = check.And(check.OK(), check.ReachedTargetClusters(t))
+
+					from.CallOrFail(t, opts)
+				})
+
+			// Rotate CA cert
+			if err := cert.CreateCustomCASecret(t,
+				"ca-cert.pem", "ca-key.pem",
+				"cert-chain.pem", "root-cert-combined.pem"); err != nil {
+				t.Errorf("failed to update combined CA secret: %v", err)
+			}
+
+			// Wait for workload cert to be updated
+			retry.UntilOrFail(t, func() bool {
+				updateTime, err := getWorkloadCertLastUpdateTime(t, from[0], istioCtl)
+				if err != nil {
+					t.Logf("failed to get workload cert last update time: %v", err)
+					return false
+				}
+
+				// retry when workload cert is not updated
+				if updateTime.After(lastUpdateTime) {
+					t.Logf("workload cert is updated, last update time: %v", updateTime)
+					return true
+				}
+
+				return false
+			}, retry.Timeout(time.Minute), retry.Delay(1*time.Second))
+
+			// Verify traffic works between a and b after cert rotation
+			echotest.New(t, fromAndTo).
+				WithDefaultFilters(1, 1).
+				FromMatch(match.ServiceName(from.NamespacedName())).
+				ToMatch(match.ServiceName(to.NamespacedName())).
+				Run(func(t framework.TestContext, from echo.Instance, to echo.Target) {
+					// Verify mTLS works between a and b
+					opts := echo.CallOptions{
+						To: to,
+						Port: echo.Port{
+							Name: "http",
+						},
+					}
+					opts.Check = check.And(check.OK(), check.ReachedTargetClusters(t))
+
+					from.CallOrFail(t, opts)
+				})
+		})
+}
+
+func getWorkloadCertLastUpdateTime(t framework.TestContext, i echo.Instance, ctl istioctl.Instance) (time.Time, error) {
+	podID, err := getPodID(i)
+	if err != nil {
+		t.Fatalf("Could not get Pod ID: %v", err)
+	}
+	podName := fmt.Sprintf("%s.%s", podID, i.NamespaceName())
+	out, errOut, err := ctl.Invoke([]string{"pc", "s", podName, "-o", "json"})
+	if err != nil || errOut != "" {
+		t.Errorf("failed to retrieve pod secret from %s, err: %v errOut: %s", podName, err, errOut)
+	}
+
+	dump := &admin.SecretsConfigDump{}
+	if err := protomarshal.Unmarshal([]byte(out), dump); err != nil {
+		t.Errorf("failed to unmarshal secret dump: %v", err)
+	}
+
+	for _, s := range dump.DynamicActiveSecrets {
+		if s.Name == security.WorkloadKeyCertResourceName {
+			return s.LastUpdated.AsTime(), nil
+		}
+	}
+
+	return time.Now(), errors.New("failed to find workload cert")
+}
+
+func getPodID(i echo.Instance) (string, error) {
+	wls, err := i.Workloads()
+	if err != nil {
+		return "", nil
+	}
+
+	for _, wl := range wls {
+		return wl.PodName(), nil
+	}
+
+	return "", fmt.Errorf("no workloads")
+}
diff --git a/tests/integration/security/util/cert/cert.go b/tests/integration/security/util/cert/cert.go
@@ -68,23 +68,29 @@ func DumpCertFromSidecar(t test.Failer, from echo.Instance, to echo.Target, port
 
 // CreateCASecret creates a k8s secret "cacerts" to store the CA key and cert.
 func CreateCASecret(ctx resource.Context) error {
+	return CreateCustomCASecret(ctx,
+		"ca-cert.pem", "ca-key.pem",
+		"cert-chain.pem", "root-cert.pem")
+}
+
+func CreateCustomCASecret(ctx resource.Context, caCertFile, caKeyFile, certChainFile, rootCertFile string) error {
 	name := "cacerts"
 	systemNs, err := istio.ClaimSystemNamespace(ctx)
 	if err != nil {
 		return err
 	}
 
 	var caCert, caKey, certChain, rootCert []byte
-	if caCert, err = ReadSampleCertFromFile("ca-cert.pem"); err != nil {
+	if caCert, err = ReadSampleCertFromFile(caCertFile); err != nil {
 		return err
 	}
-	if caKey, err = ReadSampleCertFromFile("ca-key.pem"); err != nil {
+	if caKey, err = ReadSampleCertFromFile(caKeyFile); err != nil {
 		return err
 	}
-	if certChain, err = ReadSampleCertFromFile("cert-chain.pem"); err != nil {
+	if certChain, err = ReadSampleCertFromFile(certChainFile); err != nil {
 		return err
 	}
-	if rootCert, err = ReadSampleCertFromFile("root-cert.pem"); err != nil {
+	if rootCert, err = ReadSampleCertFromFile(rootCertFile); err != nil {
 		return err
 	}