Self image lookup: Retry on empty string

Occasionally this appears to return am empty string, which then makes things fail down the line as an empty string is not a valid image. Retry if that happens. Output for when this happened: ``` {"level":"info","ts":"2022-07-28T16:55:17Z","logger":"setup","msg":"using operator image","image":""} ```
openshift · Jul 28, 2022 · b54e5ff · b54e5ff
1 parent 8e82b88
commit b54e5ff
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 5 deletions.
diff --git a/control-plane-operator/main.go b/control-plane-operator/main.go
@@ -27,6 +27,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -266,8 +267,19 @@ func NewStartCommand() *cobra.Command {
 			}
 			return "", fmt.Errorf("couldn't locate operator container on deployment")
 		}
-		hostedClusterConfigOperatorImage, err = lookupOperatorImage(hostedClusterConfigOperatorImage)
-		if err != nil {
+
+		if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
+			hostedClusterConfigOperatorImage, err = lookupOperatorImage(hostedClusterConfigOperatorImage)
+			if err != nil {
+				return false, err
+			}
+			// Apparently this is occasionally set to an empty string
+			if hostedClusterConfigOperatorImage == "" {
+				setupLog.Info("hosted cluster config operator image is empty, retrying")
+				return false, nil
+			}
+			return true, nil
+		}); err != nil {
 			setupLog.Error(err, fmt.Sprintf("failed to find operator image: %s", err), "controller", "hosted-control-plane")
 			os.Exit(1)
 		}

diff --git a/hypershift-operator/main.go b/hypershift-operator/main.go
@@ -47,6 +47,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -196,10 +197,24 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {
 		}
 		return "", fmt.Errorf("couldn't locate operator container on deployment")
 	}
-	operatorImage, err := lookupOperatorImage(opts.ControlPlaneOperatorImage)
-	if err != nil {
-		return fmt.Errorf("failed to find operator image: %w", err)
+	var operatorImage string
+	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
+		operatorImage, err = lookupOperatorImage(opts.ControlPlaneOperatorImage)
+		if err != nil {
+			return false, err
+		}
+		// Apparently this is occasionally set to an empty string
+		if operatorImage == "" {
+			log.Info("operator image is empty, retrying")
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		if err != nil {
+			return fmt.Errorf("failed to find operator image: %w", err)
+		}
 	}
+
 	log.Info("using hosted control plane operator image", "operator-image", operatorImage)
 
 	createOrUpdate := upsert.New(opts.EnableCIDebugOutput)