Skip to content

Commit

Permalink
Merge pull request kubernetes#128966 from liggitt/deflake-external
Browse files Browse the repository at this point in the history 
Wait for updated keys to be observed
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Nov 25, 2024
2 parents e4c1f98 + 26c08dd commit 8770bd5
Showing 1 changed file with 58 additions and 25 deletions.
83 changes: 58 additions & 25 deletions test/integration/serviceaccount/external_jwt_signer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,19 @@ import (
"testing"
"time"

"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/kubernetes/test/utils/ktesting"

authv1 "k8s.io/api/authentication/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics/testutil"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/features"
v1alpha1testing "k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin/testing/v1alpha1"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/kubernetes/test/utils/ktesting"
)

func TestExternalJWTSigningAndAuth(t *testing.T) {
Expand Down Expand Up @@ -94,29 +96,29 @@ func TestExternalJWTSigningAndAuth(t *testing.T) {

testCases := []struct {
desc string
preTestSignerUpdate func()
preValidationSignerUpdate func()
preTestSignerUpdate func(t *testing.T)
preValidationSignerUpdate func(t *testing.T)
wantTokenReqErr error
shouldPassAuth bool
}{
{
desc: "signing key supported.",
preTestSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func() { /*no-op*/ },
preTestSignerUpdate: func(_ *testing.T) { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
shouldPassAuth: true,
},
{
desc: "signing key not among supported set",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningKey = key1
mockSigner.SigningKeyID = "updated-kid-1"
},
preValidationSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
shouldPassAuth: false,
},
{
desc: "signing key corresponds to public key that is excluded from OIDC",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningKey = key1
mockSigner.SigningKeyID = "updated-kid-1"

Expand All @@ -130,56 +132,57 @@ func TestExternalJWTSigningAndAuth(t *testing.T) {
}
mockSigner.SetSupportedKeys(cpy)
},
preValidationSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
wantTokenReqErr: fmt.Errorf("failed to generate token: while validating header: key used for signing JWT (kid: updated-kid-1) is excluded from OIDC discovery docs"),
},
{
desc: "different signing and supported keys with same id",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningKey = key1
},
preValidationSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
shouldPassAuth: false,
},
{
desc: "token gen failure with un-supported Alg type",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningAlg = "ABC"
},
preValidationSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
wantTokenReqErr: fmt.Errorf("failed to generate token: while validating header: bad signing algorithm \"ABC\""),
},
{
desc: "token gen failure with un-supported token type",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.TokenType = "ABC"
},
preValidationSignerUpdate: func() { /*no-op*/ },
preValidationSignerUpdate: func(_ *testing.T) { /*no-op*/ },
wantTokenReqErr: fmt.Errorf("failed to generate token: while validating header: bad type"),
},
{
desc: "change of supported keys not picked immediately",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningKey = key1
},
preValidationSignerUpdate: func() {
preValidationSignerUpdate: func(_ *testing.T) {
mockSigner.SetSupportedKeys(map[string]v1alpha1testing.KeyT{})
},
shouldPassAuth: false,
},
{
desc: "change of supported keys picked up after periodic sync",
preTestSignerUpdate: func() {
preTestSignerUpdate: func(t *testing.T) {
mockSigner.SigningKey = key1
},
preValidationSignerUpdate: func() {
preValidationSignerUpdate: func(t *testing.T) {
t.Helper()
cpy := make(map[string]v1alpha1testing.KeyT)
for key, value := range mockSigner.GetSupportedKeys() {
cpy[key] = value
}
cpy["kid-1"] = v1alpha1testing.KeyT{Key: pubKey1Bytes}
mockSigner.SetSupportedKeys(cpy)
mockSigner.WaitForSupportedKeysFetch()
waitForDataTimestamp(t, client, time.Now())
},
shouldPassAuth: true,
},
Expand All @@ -195,7 +198,7 @@ func TestExternalJWTSigningAndAuth(t *testing.T) {
mockSigner.WaitForSupportedKeysFetch()

// Adjust parameters on mock signer for the test.
tc.preTestSignerUpdate()
tc.preTestSignerUpdate(t)

// Request a token for ns-1:sa-1.
tokenExpirationSec := int64(2 * 60 * 60) // 2h
Expand All @@ -214,7 +217,7 @@ func TestExternalJWTSigningAndAuth(t *testing.T) {
}

// Adjust parameters on mock signer for the test.
tc.preValidationSignerUpdate()
tc.preValidationSignerUpdate(t)

// Try Validating the token.
tokenReviewResult, err := client.AuthenticationV1().TokenReviews().Create(ctx, &authv1.TokenReview{
Expand All @@ -235,6 +238,36 @@ func TestExternalJWTSigningAndAuth(t *testing.T) {
}
}

func waitForDataTimestamp(t *testing.T, client kubernetes.Interface, minimumDataTimestamp time.Time) {
t.Helper()
minimumSample := float64(minimumDataTimestamp.UnixNano()) / float64(1000000000)
t.Logf("waiting for >=%f", minimumSample)
err := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
rawMetrics, err := client.CoreV1().RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
if err != nil {
return false, err
}
metrics := testutil.NewMetrics()
if err := testutil.ParseMetrics(string(rawMetrics), &metrics); err != nil {
return false, err
}
samples, ok := metrics["apiserver_externaljwt_fetch_keys_data_timestamp"]
if !ok || len(samples) == 0 {
t.Log("no samples found for apiserver_externaljwt_fetch_keys_data_timestamp, retrying...")
return false, nil
}
if minimumSample > float64(samples[0].Value) {
t.Logf("apiserver_externaljwt_fetch_keys_data_timestamp at %f, waiting until >=%f...", samples[0].Value, minimumSample)
return false, nil
}
t.Logf("saw %f", samples[0].Value)
return true, nil
})
if err != nil {
t.Fatal(err)
}
}

func TestDelayedStartForSigner(t *testing.T) {
// Enable feature gate for external JWT signer.
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExternalServiceAccountTokenSigner, true)
Expand Down

0 comments on commit 8770bd5

Please sign in to comment.