diff --git a/cmd/cluster/core/create.go b/cmd/cluster/core/create.go index 1c292f4245..4655781c80 100644 --- a/cmd/cluster/core/create.go +++ b/cmd/cluster/core/create.go @@ -648,7 +648,7 @@ func (opts *RawCreateOptions) Validate(ctx context.Context) (*ValidatedCreateOpt if err != nil { return nil, fmt.Errorf("could not retrieve kube clientset: %w", err) } - if err := validateMgmtClusterAndNodePoolCPUArchitectures(ctx, opts, kc); err != nil { + if err := validateMgmtClusterAndNodePoolCPUArchitectures(ctx, opts, kc, &hyperutil.RegistryClientImageMetadataProvider{}); err != nil { return nil, err } } @@ -1012,7 +1012,7 @@ func parseTolerationString(str string) (*corev1.Toleration, error) { // validateMgmtClusterAndNodePoolCPUArchitectures checks if a multi-arch release image or release stream was provided. // If none were provided, checks to make sure the NodePool CPU arch and the management cluster CPU arch match; if they // do not, the CLI will return an error since the NodePool will fail to complete during runtime. -func validateMgmtClusterAndNodePoolCPUArchitectures(ctx context.Context, opts *RawCreateOptions, kc kubeclient.Interface) error { +func validateMgmtClusterAndNodePoolCPUArchitectures(ctx context.Context, opts *RawCreateOptions, kc kubeclient.Interface, imageMetadataProvider hyperutil.ImageMetadataProvider) error { validMultiArchImage := false // Check if the release image is multi-arch @@ -1021,8 +1021,7 @@ func validateMgmtClusterAndNodePoolCPUArchitectures(ctx context.Context, opts *R if err != nil { return fmt.Errorf("failed to read pull secret file: %w", err) } - - validMultiArchImage, err = registryclient.IsMultiArchManifestList(ctx, opts.ReleaseImage, pullSecret) + validMultiArchImage, err = registryclient.IsMultiArchManifestList(ctx, opts.ReleaseImage, pullSecret, imageMetadataProvider) if err != nil { return err } diff --git a/cmd/cluster/core/create_test.go b/cmd/cluster/core/create_test.go index c48a3991f3..0f886deabb 100644 --- a/cmd/cluster/core/create_test.go +++ b/cmd/cluster/core/create_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/dockerv1client" + "github.com/openshift/hypershift/support/util/fakeimagemetadataprovider" crclient "sigs.k8s.io/controller-runtime/pkg/client" corev1 "k8s.io/api/core/v1" @@ -19,11 +21,15 @@ func TestValidateMgmtClusterAndNodePoolCPUArchitectures(t *testing.T) { fakeKubeClient := fakekubeclient.NewSimpleClientset() fakeDiscovery, ok := fakeKubeClient.Discovery().(*fakediscovery.FakeDiscovery) - if !ok { t.Fatalf("failed to convert FakeDiscovery") } + fakeMetadataProvider := &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ + Result: &dockerv1client.DockerImageConfig{}, + Manifest: fakeimagemetadataprovider.FakeManifest{}, + } + // if you want to fake a specific version fakeDiscovery.FakedServerVersion = &apiversion.Info{ Platform: "linux/amd64", @@ -80,7 +86,7 @@ func TestValidateMgmtClusterAndNodePoolCPUArchitectures(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - err := validateMgmtClusterAndNodePoolCPUArchitectures(ctx, tc.opts, fakeKubeClient) + err := validateMgmtClusterAndNodePoolCPUArchitectures(ctx, tc.opts, fakeKubeClient, fakeMetadataProvider) if tc.expectError { g.Expect(err).To(HaveOccurred()) } else { diff --git a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go index d5298c4db8..09adb0055e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go @@ -115,7 +115,7 @@ func cvoLabels() map[string]string { var port int32 = 8443 -func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, controlPlaneImage, releaseImage, cliImage, availabilityProberImage, clusterID string, updateService configv1.URL, platformType hyperv1.PlatformType, oauthEnabled, enableCVOManagementClusterMetricsAccess bool, featureSet configv1.FeatureSet) error { +func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, cliImage, availabilityProberImage, clusterID string, updateService configv1.URL, platformType hyperv1.PlatformType, oauthEnabled, enableCVOManagementClusterMetricsAccess bool, featureSet configv1.FeatureSet) error { ownerRef.ApplyTo(deployment) // preserve existing resource requirements for main CVO container @@ -138,11 +138,11 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef Spec: corev1.PodSpec{ AutomountServiceAccountToken: ptr.To(false), InitContainers: []corev1.Container{ - util.BuildContainer(cvoContainerPrepPayload(), buildCVOContainerPrepPayload(releaseImage, platformType, oauthEnabled, featureSet)), + util.BuildContainer(cvoContainerPrepPayload(), buildCVOContainerPrepPayload(dataPlaneReleaseImage, platformType, oauthEnabled, featureSet)), util.BuildContainer(cvoContainerBootstrap(), buildCVOContainerBootstrap(cliImage, clusterID)), }, Containers: []corev1.Container{ - util.BuildContainer(cvoContainerMain(), buildCVOContainerMain(controlPlaneImage, releaseImage, deployment.Namespace, updateService, enableCVOManagementClusterMetricsAccess)), + util.BuildContainer(cvoContainerMain(), buildCVOContainerMain(controlPlaneReleaseImage, dataPlaneReleaseImage, deployment.Namespace, updateService, enableCVOManagementClusterMetricsAccess)), }, Volumes: []corev1.Volume{ util.BuildVolume(cvoVolumePayload(), buildCVOVolumePayload), @@ -331,6 +331,7 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool, " release.openshift.io/delete: \"true\"", ) } + stmts = append(stmts, "EOF") return strings.Join(stmts, "\n") } @@ -361,17 +362,17 @@ oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.y return fmt.Sprintf(scriptTemplate, clusterID, payloadDir) } -func buildCVOContainerMain(image, releaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) { +func buildCVOContainerMain(controlPlaneReleaseImage, dataPlaneReleaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) { cpath := func(vol, file string) string { return path.Join(volumeMounts.Path(cvoContainerMain().Name, vol), file) } return func(c *corev1.Container) { - c.Image = image + c.Image = controlPlaneReleaseImage c.Command = []string{"cluster-version-operator"} c.Args = []string{ "start", "--release-image", - releaseImage, + dataPlaneReleaseImage, "--enable-auto-update=false", "--kubeconfig", path.Join(volumeMounts.Path(c.Name, cvoVolumeKubeconfig().Name), kas.KubeconfigKey), diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 285879f0d8..e25fdc6c6b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -175,6 +175,7 @@ type HostedControlPlaneReconciler struct { awsSession *session.Session reconcileInfrastructureStatus func(ctx context.Context, hcp *hyperv1.HostedControlPlane) (infra.InfrastructureStatus, error) EnableCVOManagementClusterMetricsAccess bool + ImageMetadataProvider util.ImageMetadataProvider IsCPOV2 bool } @@ -3639,9 +3640,47 @@ func (r *HostedControlPlaneReconciler) reconcileClusterVersionOperator(ctx conte } } + var ( + controlPlaneReleaseImage string + dataPlaneReleaseImage string + ) + // The CVO prepare-payload script needs the ReleaseImage digest for disconnected environments + pullSecret := common.PullSecret(hcp.Namespace) + if err := r.Get(ctx, client.ObjectKeyFromObject(pullSecret), pullSecret); err != nil { + return fmt.Errorf("failed to get pull secret for namespace %s: %w", hcp.Namespace, err) + } + + cpRef, err := registryclient.GetCorrectArchImage(ctx, "cluster-version-operator", p.ControlPlaneImage, pullSecret.Data[corev1.DockerConfigJsonKey], r.ImageMetadataProvider) + if err != nil { + return fmt.Errorf("failed to parse control plane release image %s: %w", cpRef, err) + } + + _, cpReleaseImageRef, err := r.ImageMetadataProvider.GetDigest(ctx, cpRef, pullSecret.Data[corev1.DockerConfigJsonKey]) + if err != nil { + return fmt.Errorf("failed to get control plane release image digest %s: %w", cpRef, err) + } + + controlPlaneReleaseImage = fmt.Sprintf("%s/%s/%s", cpReleaseImageRef.Registry, cpReleaseImageRef.Namespace, cpReleaseImageRef.NameString()) + + if p.ControlPlaneImage != hcp.Spec.ReleaseImage { + dpRef, err := registryclient.GetCorrectArchImage(ctx, "cluster-version-operator", hcp.Spec.ReleaseImage, pullSecret.Data[corev1.DockerConfigJsonKey], r.ImageMetadataProvider) + if err != nil { + return fmt.Errorf("failed to parse data plane release image %s: %w", dpRef, err) + } + + _, dpReleaseImageRef, err := r.ImageMetadataProvider.GetDigest(ctx, dpRef, pullSecret.Data[corev1.DockerConfigJsonKey]) + if err != nil { + return fmt.Errorf("failed to get data plane release image digest %s: %w", dpRef, err) + } + + dataPlaneReleaseImage = fmt.Sprintf("%s/%s/%s", dpReleaseImageRef.Registry, dpReleaseImageRef.Namespace, dpReleaseImageRef.NameString()) + } else { + dataPlaneReleaseImage = controlPlaneReleaseImage + } + deployment := manifests.ClusterVersionOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - return cvo.ReconcileDeployment(deployment, p.OwnerRef, p.DeploymentConfig, p.ControlPlaneImage, p.ReleaseImage, p.CLIImage, p.AvailabilityProberImage, p.ClusterID, hcp.Spec.UpdateService, p.PlatformType, util.HCPOAuthEnabled(hcp), r.EnableCVOManagementClusterMetricsAccess, p.FeatureSet) + return cvo.ReconcileDeployment(deployment, p.OwnerRef, p.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, p.CLIImage, p.AvailabilityProberImage, p.ClusterID, hcp.Spec.UpdateService, p.PlatformType, util.HCPOAuthEnabled(hcp), r.EnableCVOManagementClusterMetricsAccess, p.FeatureSet) }); err != nil { return fmt.Errorf("failed to reconcile cluster version operator deployment: %w", err) } @@ -3939,7 +3978,7 @@ func (r *HostedControlPlaneReconciler) reconcileOperatorLifecycleManager(ctx con var getCatalogImagesErr error olmCatalogImagesOnce.Do(func() { - catalogImages, err = olm.GetCatalogImages(ctx, *hcp, pullSecret.Data[corev1.DockerConfigJsonKey], registryclient.GetListDigest) + catalogImages, err = olm.GetCatalogImages(ctx, *hcp, pullSecret.Data[corev1.DockerConfigJsonKey], registryclient.GetListDigest, r.ImageMetadataProvider) if err != nil { getCatalogImagesErr = err return diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go index b513447769..7d2bdc524b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go @@ -1817,5 +1817,14 @@ func componentsFakeDependencies(componentName string, namespace string) []client fakeComponents = append(fakeComponents, fakeComponentTemplate.DeepCopy()) } + pullSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull-secret", Namespace: "hcp-namespace"}, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(`{}`), + }, + } + + fakeComponents = append(fakeComponents, pullSecret.DeepCopy()) + return fakeComponents } diff --git a/control-plane-operator/controllers/hostedcontrolplane/olm/catalogs.go b/control-plane-operator/controllers/hostedcontrolplane/olm/catalogs.go index d0a1ca1d04..7e393c55d7 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/olm/catalogs.go +++ b/control-plane-operator/controllers/hostedcontrolplane/olm/catalogs.go @@ -122,9 +122,9 @@ func findTagReference(tags []imagev1.TagReference, name string) *imagev1.TagRefe return nil } -func GetCatalogImages(ctx context.Context, hcp hyperv1.HostedControlPlane, pullSecret []byte, digestLister registryclient.DigestListerFN) (map[string]string, error) { +func GetCatalogImages(ctx context.Context, hcp hyperv1.HostedControlPlane, pullSecret []byte, digestLister registryclient.DigestListerFN, imageMetadataProvider util.ImageMetadataProvider) (map[string]string, error) { imageRef := hcp.Spec.ReleaseImage - imageConfig, _, _, err := registryclient.GetMetadata(ctx, imageRef, pullSecret) + imageConfig, _, _, err := imageMetadataProvider.GetMetadata(ctx, imageRef, pullSecret) if err != nil { return nil, fmt.Errorf("failed to get image metadata: %w", err) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents.yaml index 909660d5ac..db7d878f3a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents.yaml @@ -284,6 +284,7 @@ spec: annotations: include.release.openshift.io/ibm-cloud-managed: "true" release.openshift.io/delete: "true" + EOF command: - /bin/bash image: cluster-version-operator diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go index f1adc11359..a7ec5b879c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go @@ -8,9 +8,11 @@ import ( configv1 "github.com/openshift/api/config/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" hyperapi "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/config" component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/releaseinfo/registryclient" "github.com/openshift/hypershift/support/util" appsv1 "k8s.io/api/apps/v1" @@ -37,11 +39,18 @@ func (cvo *clusterVersionOperator) adaptDeployment(cpContext component.WorkloadC featureSet = cpContext.HCP.Spec.Configuration.FeatureGate.FeatureSet } + // The CVO prepare-payload script needs the ReleaseImage digest for disconnected environments + controlPlaneReleaseImage, dataPlaneReleaseImage, err := discoverCVOReleaseImages(cpContext) + if err != nil { + return fmt.Errorf("failed to discover CVO release images: %w", err) + } + util.UpdateContainer("prepare-payload", deployment.Spec.Template.Spec.InitContainers, func(c *corev1.Container) { c.Args = []string{ "-c", preparePayloadScript(cpContext.HCP.Spec.Platform.Type, util.HCPOAuthEnabled(cpContext.HCP), featureSet), } + c.Image = controlPlaneReleaseImage }) util.UpdateContainer("bootstrap", deployment.Spec.Template.Spec.InitContainers, func(c *corev1.Container) { c.Env = append(c.Env, corev1.EnvVar{ @@ -53,7 +62,7 @@ func (cvo *clusterVersionOperator) adaptDeployment(cpContext component.WorkloadC util.UpdateContainer(ComponentName, deployment.Spec.Template.Spec.Containers, func(c *corev1.Container) { util.UpsertEnvVar(c, corev1.EnvVar{ Name: "RELEASE_IMAGE", - Value: cpContext.HCP.Spec.ReleaseImage, + Value: dataPlaneReleaseImage, }) if updateService := cpContext.HCP.Spec.UpdateService; updateService != "" { @@ -210,6 +219,7 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool, " release.openshift.io/delete: \"true\"", ) } + stmts = append(stmts, "EOF") return strings.Join(stmts, "\n") } @@ -239,3 +249,45 @@ func resourcesToRemove(platformType hyperv1.PlatformType) []client.Object { } } } + +func discoverCVOReleaseImages(cpContext component.WorkloadContext) (string, string, error) { + var ( + controlPlaneReleaseImage string + dataPlaneReleaseImage string + ) + + pullSecret := common.PullSecret(cpContext.HCP.Namespace) + if err := cpContext.Client.Get(cpContext.Context, client.ObjectKeyFromObject(pullSecret), pullSecret); err != nil { + return "", "", fmt.Errorf("failed to get pull secret for namespace %s: %w", cpContext.HCP.Namespace, err) + } + + cpRef, err := registryclient.GetCorrectArchImage(cpContext.Context, "cluster-version-operator", cpContext.HCP.Spec.ReleaseImage, pullSecret.Data[corev1.DockerConfigJsonKey], cpContext.ImageMetadataProvider) + if err != nil { + return "", "", fmt.Errorf("failed to parse control plane release image %s: %w", cpRef, err) + } + + _, cpReleaseImageRef, err := cpContext.ImageMetadataProvider.GetDigest(cpContext.Context, cpRef, pullSecret.Data[corev1.DockerConfigJsonKey]) + if err != nil { + return "", "", fmt.Errorf("failed to get control plane release image digest %s: %w", cpRef, err) + } + + controlPlaneReleaseImage = fmt.Sprintf("%s/%s/%s", cpReleaseImageRef.Registry, cpReleaseImageRef.Namespace, cpReleaseImageRef.NameString()) + + if cpContext.HCP.Spec.ControlPlaneReleaseImage != nil && *cpContext.HCP.Spec.ControlPlaneReleaseImage != cpContext.HCP.Spec.ReleaseImage { + dpRef, err := registryclient.GetCorrectArchImage(cpContext.Context, "cluster-version-operator", cpContext.HCP.Spec.ReleaseImage, pullSecret.Data[corev1.DockerConfigJsonKey], cpContext.ImageMetadataProvider) + if err != nil { + return "", "", fmt.Errorf("failed to parse data plane release image %s: %w", dpRef, err) + } + + _, dpReleaseImageRef, err := cpContext.ImageMetadataProvider.GetDigest(cpContext.Context, dpRef, pullSecret.Data[corev1.DockerConfigJsonKey]) + if err != nil { + return "", "", fmt.Errorf("failed to get data plane release image digest %s: %w", dpRef, err) + } + + dataPlaneReleaseImage = fmt.Sprintf("%s/%s/%s", dpReleaseImageRef.Registry, dpReleaseImageRef.Namespace, dpReleaseImageRef.NameString()) + } else { + dataPlaneReleaseImage = controlPlaneReleaseImage + } + + return controlPlaneReleaseImage, dataPlaneReleaseImage, nil +} diff --git a/control-plane-operator/hostedclusterconfigoperator/cmd.go b/control-plane-operator/hostedclusterconfigoperator/cmd.go index 32409ca40f..f317622371 100644 --- a/control-plane-operator/hostedclusterconfigoperator/cmd.go +++ b/control-plane-operator/hostedclusterconfigoperator/cmd.go @@ -259,6 +259,10 @@ func (o *HostedClusterConfigOperator) Run(ctx context.Context) error { OpenShiftImageRegistryOverrides: imageRegistryOverrides, } + imageMetaDataProvider := &util.RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: imageRegistryOverrides, + } + apiReadingClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: hyperapi.Scheme}) if err != nil { return fmt.Errorf("failed to construct api reading client: %w", err) @@ -299,6 +303,7 @@ func (o *HostedClusterConfigOperator) Run(ctx context.Context) error { OAuthPort: o.OAuthPort, OperateOnReleaseImage: os.Getenv("OPERATE_ON_RELEASE_IMAGE"), EnableCIDebugOutput: o.enableCIDebugOutput, + ImageMetaDataProvider: *imageMetaDataProvider, } configmetrics.Register(mgr.GetCache()) return operatorConfig.Start(ctx) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/olm/params.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/olm/params.go index 5301857c48..8c213df2c1 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/olm/params.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/olm/params.go @@ -3,9 +3,11 @@ package olm import ( "context" "fmt" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/olm" "github.com/openshift/hypershift/support/releaseinfo/registryclient" + "github.com/openshift/hypershift/support/util" corev1 "k8s.io/api/core/v1" ) @@ -17,8 +19,8 @@ type OperatorLifecycleManagerParams struct { OLMCatalogPlacement hyperv1.OLMCatalogPlacement } -func NewOperatorLifecycleManagerParams(ctx context.Context, hcp *hyperv1.HostedControlPlane, pullSecret *corev1.Secret, digestLister registryclient.DigestListerFN) (*OperatorLifecycleManagerParams, error) { - catalogImages, err := olm.GetCatalogImages(ctx, *hcp, pullSecret.Data[corev1.DockerConfigJsonKey], digestLister) +func NewOperatorLifecycleManagerParams(ctx context.Context, hcp *hyperv1.HostedControlPlane, pullSecret *corev1.Secret, digestLister registryclient.DigestListerFN, imageMetadataProvider util.ImageMetadataProvider) (*OperatorLifecycleManagerParams, error) { + catalogImages, err := olm.GetCatalogImages(ctx, *hcp, pullSecret.Data[corev1.DockerConfigJsonKey], digestLister, imageMetadataProvider) if err != nil { return nil, fmt.Errorf("failed to get catalog images: %w", err) } diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 94f8cd73a6..4a657aec8f 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -131,6 +131,7 @@ type reconciler struct { oauthPort int32 versions map[string]string operateOnReleaseImage string + ImageMetaDataProvider util.RegistryClientImageMetadataProvider registryclient.DigestListerFN } @@ -190,6 +191,7 @@ func Setup(ctx context.Context, opts *operator.HostedClusterConfigOperatorConfig versions: opts.Versions, operateOnReleaseImage: opts.OperateOnReleaseImage, DigestListerFN: registryclient.GetListDigest, + ImageMetaDataProvider: opts.ImageMetaDataProvider, }}) if err != nil { return fmt.Errorf("failed to construct controller: %w", err) @@ -1687,7 +1689,7 @@ func (r *reconciler) reconcileOLM(ctx context.Context, hcp *hyperv1.HostedContro olmCatalogImagesOnce.Do(func() { var err error - p, err = olm.NewOperatorLifecycleManagerParams(ctx, hcp, pullSecret, digestLister) + p, err = olm.NewOperatorLifecycleManagerParams(ctx, hcp, pullSecret, digestLister, &r.ImageMetaDataProvider) if err != nil { errs = append(errs, fmt.Errorf("failed to create OperatorLifecycleManagerParams: %w", err)) return diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go index 99d4028986..52a3e1b95d 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go @@ -141,6 +141,9 @@ func TestReconcileErrorHandling(t *testing.T) { fakeDigestLister := func(ctx context.Context, image string, pullSecret []byte) (digest.Digest, error) { return "", nil } + imageMetaDataProvider := supportutil.RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: map[string][]string{}, + } r := &reconciler{ client: fakeClient, @@ -153,6 +156,7 @@ func TestReconcileErrorHandling(t *testing.T) { hcpNamespace: "bar", releaseProvider: &fakereleaseprovider.FakeReleaseProvider{}, DigestListerFN: fakeDigestLister, + ImageMetaDataProvider: imageMetaDataProvider, } _, err := r.Reconcile(context.Background(), controllerruntime.Request{}) if err != nil { diff --git a/control-plane-operator/hostedclusterconfigoperator/operator/config.go b/control-plane-operator/hostedclusterconfigoperator/operator/config.go index 6dee9f0cb4..7410810bcd 100644 --- a/control-plane-operator/hostedclusterconfigoperator/operator/config.go +++ b/control-plane-operator/hostedclusterconfigoperator/operator/config.go @@ -30,6 +30,7 @@ import ( "github.com/openshift/hypershift/support/releaseinfo" "github.com/openshift/hypershift/support/releaseinfo/registryclient" "github.com/openshift/hypershift/support/upsert" + "github.com/openshift/hypershift/support/util" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" ) @@ -73,6 +74,7 @@ type HostedClusterConfigOperatorConfig struct { OperateOnReleaseImage string EnableCIDebugOutput bool ListDigestsFN registryclient.DigestListerFN + ImageMetaDataProvider util.RegistryClientImageMetadataProvider kubeClient kubeclient.Interface } diff --git a/control-plane-operator/main.go b/control-plane-operator/main.go index 5e9aec3aa0..19a4e7ecc4 100644 --- a/control-plane-operator/main.go +++ b/control-plane-operator/main.go @@ -379,7 +379,6 @@ func NewStartCommand() *cobra.Command { "token-minter": tokenMinterImage, util.CPOImageName: cpoImage, util.CPPKIOImageName: cpoImage, - "cluster-version-operator": os.Getenv("OPERATE_ON_RELEASE_IMAGE"), } for name, image := range imageOverrides { componentImages[name] = image @@ -418,6 +417,10 @@ func NewStartCommand() *cobra.Command { OpenShiftImageRegistryOverrides: imageRegistryOverrides, } + imageMetaDataProvider := &util.RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: imageRegistryOverrides, + } + defaultIngressDomain := os.Getenv(config.DefaultIngressDomainEnvVar) metricsSet, err := metrics.MetricsSetFromEnv() @@ -442,6 +445,7 @@ func NewStartCommand() *cobra.Command { CertRotationScale: certRotationScale, EnableCVOManagementClusterMetricsAccess: enableCVOManagementClusterMetricsAccess, IsCPOV2: isCPOV2, + ImageMetadataProvider: imageMetaDataProvider, }).SetupWithManager(mgr, upsert.New(enableCIDebugOutput).CreateOrUpdate); err != nil { setupLog.Error(err, "unable to create controller", "controller", "hosted-control-plane") os.Exit(1) diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go index 1d766fab87..e0982e3f7d 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go @@ -10,6 +10,8 @@ import ( "github.com/openshift/hypershift/cmd/util" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" @@ -30,6 +32,7 @@ import ( "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/releaseinfo" fakereleaseprovider "github.com/openshift/hypershift/support/releaseinfo/fake" + "github.com/openshift/hypershift/support/releaseinfo/registryclient" "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/dockerv1client" "github.com/openshift/hypershift/support/upsert" hyperutil "github.com/openshift/hypershift/support/util" @@ -57,8 +60,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -var Now = metav1.NewTime(time.Now()) -var Later = metav1.NewTime(Now.Add(5 * time.Minute)) +var ( + Now = metav1.NewTime(time.Now()) + Later = metav1.NewTime(Now.Add(5 * time.Minute)) +) + +const ( + ManifestListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" + LinuxOS = "linux" + ArchitectureAMD64 = "amd64" + ArchitecturePPC64LE = "ppc64le" +) func TestHasBeenAvailable(t *testing.T) { now := time.Now().Truncate(time.Second) @@ -155,7 +167,10 @@ func TestHasBeenAvailable(t *testing.T) { createOrUpdate: func(reconcile.Request) upsert.CreateOrUpdateFN { return ctrl.CreateOrUpdate }, ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, ReconcileMetadataProviders: func(ctx context.Context, imgOverrides map[string]string) (releaseinfo.ProviderWithOpenShiftImageRegistryOverrides, hyperutil.ImageMetadataProvider, error) { - return &fakereleaseprovider.FakeReleaseProvider{}, &fakeimagemetadataprovider.FakeImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{}}, nil + return &fakereleaseprovider.FakeReleaseProvider{}, &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ + Result: &dockerv1client.DockerImageConfig{}, + Manifest: fakeimagemetadataprovider.FakeManifest{}, + }, nil }, now: func() metav1.Time { return reconcilerNow }, } @@ -899,6 +914,35 @@ func expectedRules(addRules []rbacv1.PolicyRule) []rbacv1.PolicyRule { func TestHostedClusterWatchesEverythingItCreates(t *testing.T) { releaseImage, _ := version.LookupDefaultOCPVersion("") + manifests := []manifestlist.ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestListMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitectureAMD64, + OS: LinuxOS, + }, + }, + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestListMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitecturePPC64LE, + OS: LinuxOS, + }, + }, + } + deserializeFunc := func(payload []byte) (*manifestlist.DeserializedManifestList, error) { + return &manifestlist.DeserializedManifestList{ + ManifestList: manifestlist.ManifestList{ + Manifests: manifests, + }, + }, nil + } hostedClusters := []*hyperv1.HostedCluster{ { ObjectMeta: metav1.ObjectMeta{ @@ -1083,7 +1127,12 @@ func TestHostedClusterWatchesEverythingItCreates(t *testing.T) { ), createOrUpdate: func(reconcile.Request) upsert.CreateOrUpdateFN { return ctrl.CreateOrUpdate }, ReconcileMetadataProviders: func(ctx context.Context, imgOverrides map[string]string) (releaseinfo.ProviderWithOpenShiftImageRegistryOverrides, hyperutil.ImageMetadataProvider, error) { - return &fakereleaseprovider.FakeReleaseProvider{}, &fakeimagemetadataprovider.FakeImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{}}, nil + return &fakereleaseprovider.FakeReleaseProvider{}, + &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ + MediaType: ManifestListMediaType, + Result: &dockerv1client.DockerImageConfig{}, + Manifest: fakeimagemetadataprovider.FakeManifest{}, + }, nil }, EnableEtcdRecovery: true, now: metav1.Now, @@ -1099,7 +1148,8 @@ func TestHostedClusterWatchesEverythingItCreates(t *testing.T) { for _, hc := range hostedClusters { t.Run(hc.Name, func(t *testing.T) { - _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: hc.Namespace, Name: hc.Name}}) + ctx := context.WithValue(context.Background(), registryclient.DeserializeFuncName, deserializeFunc) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: hc.Namespace, Name: hc.Name}}) if err != nil { t.Fatalf("Reconcile failed: %v", err) } @@ -1910,7 +1960,7 @@ func TestValidateReleaseImage(t *testing.T) { "image-4.18.0": "4.18.0", }, }, - &fakeimagemetadataprovider.FakeImageMetadataProvider{ + &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ Result: &dockerv1client.DockerImageConfig{}, }, nil @@ -2252,7 +2302,7 @@ func TestIsUpgradeable(t *testing.T) { "image-4.14": "4.15.0", }, }, - &fakeimagemetadataprovider.FakeImageMetadataProvider{ + &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ Result: &dockerv1client.DockerImageConfig{}, }, nil @@ -2609,7 +2659,7 @@ func TestIsProgressing(t *testing.T) { "release-1.3": "1.3.0", }, }, - &fakeimagemetadataprovider.FakeImageMetadataProvider{ + &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{ Result: &dockerv1client.DockerImageConfig{}, }, nil @@ -3535,7 +3585,7 @@ func TestKubevirtETCDEncKey(t *testing.T) { ), createOrUpdate: func(reconcile.Request) upsert.CreateOrUpdateFN { return ctrl.CreateOrUpdate }, ReconcileMetadataProviders: func(ctx context.Context, imgOverrides map[string]string) (releaseinfo.ProviderWithOpenShiftImageRegistryOverrides, hyperutil.ImageMetadataProvider, error) { - return &fakereleaseprovider.FakeReleaseProvider{}, &fakeimagemetadataprovider.FakeImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{}}, nil + return &fakereleaseprovider.FakeReleaseProvider{}, &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{}}, nil }, now: metav1.Now, } diff --git a/hypershift-operator/controllers/nodepool/secret_janitor_test.go b/hypershift-operator/controllers/nodepool/secret_janitor_test.go index 9891d50aa3..4ef68c0a3b 100644 --- a/hypershift-operator/controllers/nodepool/secret_janitor_test.go +++ b/hypershift-operator/controllers/nodepool/secret_janitor_test.go @@ -156,7 +156,7 @@ spec: //We need the ReleaseProvider to stay at 4.18 so that the token doesn't get updated when bumping releases, // this protects us from possibly hiding other factors that might be causing the token to be updated ReleaseProvider: &fakereleaseprovider.FakeReleaseProvider{Version: semver.MustParse("4.18.0").String()}, - ImageMetadataProvider: &fakeimagemetadataprovider.FakeImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{Config: &docker10.DockerConfig{ + ImageMetadataProvider: &fakeimagemetadataprovider.FakeRegistryClientImageMetadataProvider{Result: &dockerv1client.DockerImageConfig{Config: &docker10.DockerConfig{ Labels: map[string]string{}, }}}, }, diff --git a/ignition-server/cmd/run_local_ignitionprovider.go b/ignition-server/cmd/run_local_ignitionprovider.go index 387e8073b4..e6f4390c75 100644 --- a/ignition-server/cmd/run_local_ignitionprovider.go +++ b/ignition-server/cmd/run_local_ignitionprovider.go @@ -92,15 +92,20 @@ func (o *RunLocalIgnitionProviderOptions) Run(ctx context.Context) error { return fmt.Errorf("unable to create image file cache: %w", err) } + imageMetaDataProvider := &util.RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: map[string][]string{}, + } + p := &controllers.LocalIgnitionProvider{ - Client: cl, - ReleaseProvider: &releaseinfo.ProviderWithOpenShiftImageRegistryOverridesDecorator{}, - CloudProvider: "", - Namespace: o.Namespace, - WorkDir: o.WorkDir, - PreserveOutput: true, - ImageFileCache: imageFileCache, - FeatureGateManifest: o.FeatureGateManifest, + Client: cl, + ReleaseProvider: &releaseinfo.ProviderWithOpenShiftImageRegistryOverridesDecorator{}, + ImageMetadataProvider: imageMetaDataProvider, + CloudProvider: "", + Namespace: o.Namespace, + WorkDir: o.WorkDir, + PreserveOutput: true, + ImageFileCache: imageFileCache, + FeatureGateManifest: o.FeatureGateManifest, } payload, err := p.GetPayload(ctx, o.Image, config.String(), "", "", "") diff --git a/ignition-server/cmd/start.go b/ignition-server/cmd/start.go index 618e2509d0..316572fae4 100644 --- a/ignition-server/cmd/start.go +++ b/ignition-server/cmd/start.go @@ -149,6 +149,10 @@ func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[stri return nil, fmt.Errorf("unable to create image file cache: %w", err) } + imageMetaDataProvider := &util.RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: util.ConvertImageRegistryOverrideStringToMap(os.Getenv("OPENSHIFT_IMG_OVERRIDES")), + } + if err = (&controllers.TokenSecretReconciler{ Client: mgr.GetClient(), PayloadStore: payloadStore, @@ -163,12 +167,13 @@ func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[stri }, OpenShiftImageRegistryOverrides: util.ConvertImageRegistryOverrideStringToMap(os.Getenv("OPENSHIFT_IMG_OVERRIDES")), }, - Client: mgr.GetClient(), - Namespace: os.Getenv(namespaceEnvVariableName), - CloudProvider: cloudProvider, - WorkDir: cacheDir, - ImageFileCache: imageFileCache, - FeatureGateManifest: featureGateManifest, + Client: mgr.GetClient(), + Namespace: os.Getenv(namespaceEnvVariableName), + CloudProvider: cloudProvider, + WorkDir: cacheDir, + ImageFileCache: imageFileCache, + FeatureGateManifest: featureGateManifest, + ImageMetadataProvider: imageMetaDataProvider, }, }).SetupWithManager(ctx, mgr); err != nil { return nil, fmt.Errorf("unable to create controller: %w", err) diff --git a/ignition-server/controllers/local_ignitionprovider.go b/ignition-server/controllers/local_ignitionprovider.go index 39bf5f25d8..6b4970d594 100644 --- a/ignition-server/controllers/local_ignitionprovider.go +++ b/ignition-server/controllers/local_ignitionprovider.go @@ -72,6 +72,10 @@ type LocalIgnitionProvider struct { // to render the ignition payload. FeatureGateManifest string + // ImageMetaDataProvider is used to get the image metadata for the images + // used in the ignition payload. + ImageMetadataProvider *util.RegistryClientImageMetadataProvider + ImageFileCache *imageFileCache lock sync.Mutex @@ -191,11 +195,24 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, cu return nil, fmt.Errorf("release image does not contain machine-config-operator (images: %v)", imageProvider.ComponentImages()) } - mcoImage, err = registryclient.GetCorrectArchImage(ctx, component, mcoImage, pullSecret) + mcoImage, err = registryclient.GetCorrectArchImage(ctx, component, mcoImage, pullSecret, p.ImageMetadataProvider) + if err != nil { + return nil, err + } + + log.Info(fmt.Sprintf("discovered image %s image %v", component, mcoImage)) + + // Making sure image uses the registry override for disconnected environments + checkedMcoImage, err := p.ImageMetadataProvider.GetOverride(ctx, mcoImage, pullSecret) if err != nil { return nil, err } - log.Info("discovered machine-config-operator image", "image", mcoImage) + + mcoComposedImage := fmt.Sprintf("%s/%s/%s", checkedMcoImage.Registry, checkedMcoImage.Namespace, checkedMcoImage.NameString()) + if mcoComposedImage != mcoImage { + mcoImage = mcoComposedImage + log.Info(fmt.Sprintf("using mirrored %s image %v", component, mcoImage)) + } // Set up the base working directory workDir, err := os.MkdirTemp(p.WorkDir, "get-payload") @@ -329,12 +346,24 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, cu return fmt.Errorf("release image does not contain $%s (images: %v)", clusterConfigComponent, imageProvider.ComponentImages()) } - clusterConfigImage, err = registryclient.GetCorrectArchImage(ctx, clusterConfigComponent, clusterConfigImage, pullSecret) + clusterConfigImage, err = registryclient.GetCorrectArchImage(ctx, clusterConfigComponent, clusterConfigImage, pullSecret, p.ImageMetadataProvider) if err != nil { return err } - log.Info(fmt.Sprintf("discovered image %s image %v", clusterConfigComponent, clusterConfigImage)) + log.Info(fmt.Sprintf("discovered image %s image %v", clusterConfigComponent, clusterConfigImage)) + + // Making sure image uses the registry override for disconnected environments + checkedClusterConfigImage, err := p.ImageMetadataProvider.GetOverride(ctx, clusterConfigImage, pullSecret) + if err != nil { + return err + } + + ccaComposedImage := fmt.Sprintf("%s/%s/%s", checkedClusterConfigImage.Registry, checkedClusterConfigImage.Namespace, checkedClusterConfigImage.NameString()) + if ccaComposedImage != clusterConfigImage { + clusterConfigImage = ccaComposedImage + log.Info(fmt.Sprintf("using mirrored %s image %v", clusterConfigComponent, ccaComposedImage)) + } file, err := os.Create(filepath.Join(binDir, clusterConfigComponent)) if err != nil { @@ -750,7 +779,7 @@ EOF --asset-input-dir %[2]s/input \ --asset-output-dir %[2]s/output \ --rendered-manifest-files=%[2]s/manifests \ - --payload-version=%[4]s + --payload-version=%[4]s cp %[2]s/manifests/99_feature-gate.yaml %[3]s/99_feature-gate.yaml ` } diff --git a/support/releaseinfo/registryclient/client.go b/support/releaseinfo/registryclient/client.go index f612bf1e71..db102f744e 100644 --- a/support/releaseinfo/registryclient/client.go +++ b/support/releaseinfo/registryclient/client.go @@ -33,8 +33,17 @@ const ( ArchitectureS390X = "s390x" ArchitecturePPC64LE = "ppc64le" ArchitectureARM64 = "arm64" + + DeserializeFuncName deserializeFuncCtxKey = "deserializeFunc" ) +type deserializeFuncCtxKey string +type ManifestProvider interface { + GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) + ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) + GetMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) +} + // ExtractImageFiles extracts a list of files from a registry image given the image reference, pull secret and the // list of files to extract. It returns a map with file contents or an error. func ExtractImageFiles(ctx context.Context, imageRef string, pullSecret []byte, files ...string) (map[string][]byte, error) { @@ -281,42 +290,9 @@ func GetRepoSetup(ctx context.Context, imageRef string, pullSecret []byte) (dist return repo, dockerImageRef, nil } -// GetManifest gets the manifest from an image -func GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) { - repo, ref, err := GetRepoSetup(ctx, imageRef, pullSecret) - if err != nil { - return nil, err - } - - var srcDigest digest.Digest - if len(ref.Tag) > 0 { - desc, err := repo.Tags(ctx).Get(ctx, ref.Tag) - if err != nil { - return nil, err - } - srcDigest = desc.Digest - } - - if len(ref.ID) > 0 { - srcDigest = digest.Digest(ref.ID) - } - - manifests, err := repo.Manifests(ctx) - if err != nil { - return nil, err - } - - digestsManifest, err := manifests.Get(ctx, srcDigest, manifest.PreferManifestList) - if err != nil { - return nil, err - } - - return digestsManifest, nil -} - // IsMultiArchManifestList determines whether an image is a manifest listed image and contains manifests the following processor architectures: amd64, arm64, s390x, ppc64le -func IsMultiArchManifestList(ctx context.Context, imageRef string, pullSecret []byte) (bool, error) { - srcManifest, err := GetManifest(ctx, imageRef, pullSecret) +func IsMultiArchManifestList(ctx context.Context, imageRef string, pullSecret []byte, imageMetadataProvider ManifestProvider) (bool, error) { + srcManifest, err := imageMetadataProvider.GetManifest(ctx, imageRef, pullSecret) if err != nil { return false, fmt.Errorf("failed to retrieve manifest %s: %w", imageRef, err) } @@ -332,13 +308,19 @@ func IsMultiArchManifestList(ctx context.Context, imageRef string, pullSecret [] return false, nil } - deserializedManifestList := new(manifestlist.DeserializedManifestList) - if err = deserializedManifestList.UnmarshalJSON(payload); err != nil { - return false, fmt.Errorf("failed to get unmarshalled manifest list: %w", err) + // Default to using the deserializeManifest function, but allow for a custom deserialization function to be passed in the context for testing purposes and avoiding paralelism issues + deserializeFunc := deserializeManifest + if ctx.Value(DeserializeFuncName) != nil { + deserializeFunc = ctx.Value(DeserializeFuncName).(func([]byte) (*manifestlist.DeserializedManifestList, error)) + } + + manifestList, err := deserializeFunc(payload) + if err != nil { + return false, fmt.Errorf("failed to deserialize payload: %w", err) } count := 0 - for _, arch := range deserializedManifestList.ManifestList.Manifests { + for _, arch := range manifestList.ManifestList.Manifests { switch arch.Platform.Architecture { case ArchitectureAMD64, ArchitectureS390X, ArchitecturePPC64LE, ArchitectureARM64: count = count + 1 @@ -351,9 +333,18 @@ func IsMultiArchManifestList(ctx context.Context, imageRef string, pullSecret [] return false, nil } +func deserializeManifest(b []byte) (*manifestlist.DeserializedManifestList, error) { + deserializedManifestList := new(manifestlist.DeserializedManifestList) + if err := deserializedManifestList.UnmarshalJSON(b); err != nil { + return nil, fmt.Errorf("failed to get unmarshalled manifest list: %w", err) + } + + return deserializedManifestList, nil +} + // findImageRefByArch finds the appropriate image reference in a multi-arch manifest image based on the current platform's OS and processor architecture -func findImageRefByArch(ctx context.Context, imageRef string, pullSecret []byte, osToFind string, archToFind string) (manifestImageRef string, err error) { - manifestList, err := GetManifest(ctx, imageRef, pullSecret) +func findImageRefByArch(ctx context.Context, imageRef string, pullSecret []byte, osToFind string, archToFind string, imageMetadataPorvider ManifestProvider) (manifestImageRef string, err error) { + manifestList, err := imageMetadataPorvider.GetManifest(ctx, imageRef, pullSecret) if err != nil { return "", fmt.Errorf("failed to retrieve manifest from image ref, %s: %w", imageRef, err) } @@ -422,10 +413,10 @@ func findMatchingManifest(ctx context.Context, imageRef string, deserializedMani // GetCorrectArchImage returns the appropriate image related to the system os/arch if the image reference is manifest // listed, else returns the original image reference -func GetCorrectArchImage(ctx context.Context, component string, imageRef string, pullSecret []byte) (manifestImageRef string, err error) { +func GetCorrectArchImage(ctx context.Context, component string, imageRef string, pullSecret []byte, imageMetadataProvider ManifestProvider) (manifestImageRef string, err error) { log := ctrl.LoggerFrom(ctx) - isMultiArchImage, err := IsMultiArchManifestList(ctx, imageRef, pullSecret) + isMultiArchImage, err := IsMultiArchManifestList(ctx, imageRef, pullSecret, imageMetadataProvider) if err != nil { return "", fmt.Errorf("failed to determine if image is manifest listed: %w", err) } @@ -436,7 +427,7 @@ func GetCorrectArchImage(ctx context.Context, component string, imageRef string, log.Info(component + " image is a manifest listed image; extracting manifest for os/arch: " + operatingSystem + "/" + arch) // Verify MF Image has the right os/arch image - imageRef, err = findImageRefByArch(ctx, imageRef, pullSecret, operatingSystem, arch) + imageRef, err = findImageRefByArch(ctx, imageRef, pullSecret, operatingSystem, arch, imageMetadataProvider) if err != nil { return "", fmt.Errorf("failed to extract appropriate os/arch manifest from %s: %w", imageRef, err) } diff --git a/support/releaseinfo/registryclient/client_test.go b/support/releaseinfo/registryclient/client_test.go index eff0ef6ff4..20b86bd8c5 100644 --- a/support/releaseinfo/registryclient/client_test.go +++ b/support/releaseinfo/registryclient/client_test.go @@ -2,20 +2,69 @@ package registryclient import ( "context" + "fmt" "testing" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" + "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/dockerv1client" + "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/reference" ) const ( - ReleaseImage1 = "quay.io/openshift-release-dev/ocp-release@sha256:1a101ef5215da468cea8bd2eb47114e85b2b64a6b230d5882f845701f55d057f" - ReleaseImage2 = "quay.io/openshift-release-dev/ocp-release:4.11.0-0.nightly-multi-2022-07-12-131716" - ManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json" - LinuxOS = "linux" + ReleaseImage1 = "quay.io/openshift-release-dev/ocp-release@sha256:1a101ef5215da468cea8bd2eb47114e85b2b64a6b230d5882f845701f55d057f" + ReleaseImage2 = "quay.io/openshift-release-dev/ocp-release:4.11.0-0.nightly-multi-2022-07-12-131716" + ManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json" + ManifestListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" + ImageIndexMediaType = "application/vnd.oci.image.index.v1+json" + LinuxOS = "linux" ) +type fakeRegistryClientImageMetadataProvider struct { + mediaType string + result *dockerv1client.DockerImageConfig + digest string + ref *reference.DockerImageReference +} +type fakeManifest struct { + mediaType string +} + +func (f *fakeRegistryClientImageMetadataProvider) ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) { + return f.result, nil +} + +func (f *fakeRegistryClientImageMetadataProvider) GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) { + _, _, err := GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return nil, fmt.Errorf("failed to retrieve manifest %s: %w", imageRef, err) + } + return &fakeManifest{ + f.mediaType, + }, nil +} + +func (f *fakeRegistryClientImageMetadataProvider) GetDigest(ctx context.Context, imageRef string, pullSecret []byte) (digest.Digest, *reference.DockerImageReference, error) { + var err error + _, f.ref, err = GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return "", nil, fmt.Errorf("failed to retrieve manifest %s: %w", imageRef, err) + } + f.ref.ID = f.digest + return digest.Digest(f.digest), f.ref, nil +} + +func (f *fakeRegistryClientImageMetadataProvider) GetMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) { + return f.result, []distribution.Descriptor{}, nil, nil +} + +type FakeManifest struct{} + +func (f *fakeManifest) References() []distribution.Descriptor { return []distribution.Descriptor{} } +func (f *fakeManifest) Payload() (string, []byte, error) { return f.mediaType, []byte{}, nil } + func TestFindMatchingManifest(t *testing.T) { deserializedManifestList1 := &manifestlist.DeserializedManifestList{ ManifestList: manifestlist.ManifestList{ @@ -198,48 +247,123 @@ func TestFindMatchingManifest(t *testing.T) { func TestIsMultiArchManifestList(t *testing.T) { pullSecretBytes := []byte("{\"auths\":{\"quay.io\":{\"auth\":\"\",\"email\":\"\"}}}") - testCases := []struct { name string image string pullSecretBytes []byte + mediaType string + manifests []manifestlist.ManifestDescriptor expectedMultiArchImage bool expectErr bool }{ { name: "Check an amd64 image; no err", image: "quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64", + mediaType: ManifestMediaType, pullSecretBytes: pullSecretBytes, expectedMultiArchImage: false, expectErr: false, + manifests: []manifestlist.ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitectureAMD64, + OS: LinuxOS, + }, + }, + }, }, { name: "Check a ppc64le image; no err", image: "quay.io/openshift-release-dev/ocp-release:4.16.11-ppc64le", + mediaType: ManifestMediaType, pullSecretBytes: pullSecretBytes, expectedMultiArchImage: false, expectErr: false, + manifests: []manifestlist.ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitecturePPC64LE, + OS: LinuxOS, + }, + }, + }, }, { name: "Check a multi-arch image; no err", image: "quay.io/openshift-release-dev/ocp-release:4.16.11-multi", + mediaType: ManifestListMediaType, pullSecretBytes: pullSecretBytes, expectedMultiArchImage: true, expectErr: false, + manifests: []manifestlist.ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestListMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitectureAMD64, + OS: LinuxOS, + }, + }, + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestListMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitecturePPC64LE, + OS: LinuxOS, + }, + }, + }, }, { name: "Bad pull secret; err", image: "quay.io/openshift-release-dev/ocp-release:4.16.11-ppc64le", + mediaType: ManifestMediaType, pullSecretBytes: []byte(""), expectedMultiArchImage: false, expectErr: true, + manifests: []manifestlist.ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + MediaType: ManifestMediaType, + Digest: "sha256:70fb4524d21e1b6c08477eb5d1ca2cf282b3270b1d008f70dd7e1cf13d8ba4ce", + }, + Platform: manifestlist.PlatformSpec{ + Architecture: ArchitecturePPC64LE, + OS: LinuxOS, + }, + }, + }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - isMultiArchImage, err := IsMultiArchManifestList(context.TODO(), tc.image, tc.pullSecretBytes) + deserializeFunc := func(payload []byte) (*manifestlist.DeserializedManifestList, error) { + return &manifestlist.DeserializedManifestList{ + ManifestList: manifestlist.ManifestList{ + Manifests: tc.manifests, + }, + }, nil + } + ctx := context.WithValue(context.Background(), DeserializeFuncName, deserializeFunc) + imageMetadataProvider := &fakeRegistryClientImageMetadataProvider{ + mediaType: tc.mediaType, + } + + isMultiArchImage, err := IsMultiArchManifestList(ctx, tc.image, tc.pullSecretBytes, imageMetadataProvider) if tc.expectErr { g.Expect(err).To(HaveOccurred()) } else { diff --git a/support/util/fakeimagemetadataprovider/fakeimagemetadataprovider.go b/support/util/fakeimagemetadataprovider/fakeimagemetadataprovider.go index e17c5b6ed8..65cdbe7a40 100644 --- a/support/util/fakeimagemetadataprovider/fakeimagemetadataprovider.go +++ b/support/util/fakeimagemetadataprovider/fakeimagemetadataprovider.go @@ -2,14 +2,63 @@ package fakeimagemetadataprovider import ( "context" + "fmt" + "github.com/docker/distribution" + "github.com/opencontainers/go-digest" + "github.com/openshift/hypershift/support/releaseinfo/registryclient" "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/dockerv1client" + "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/reference" ) -type FakeImageMetadataProvider struct { - Result *dockerv1client.DockerImageConfig +type FakeImageMetadataProvider interface { + ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) + GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) + GetDigest(ctx context.Context, imageRef string, pullSecret []byte) (digest.Digest, *reference.DockerImageReference, error) } -func (f *FakeImageMetadataProvider) ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) { +func (f *FakeRegistryClientImageMetadataProvider) ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) { return f.Result, nil } + +func (f *FakeManifest) References() []distribution.Descriptor { return []distribution.Descriptor{} } +func (f *FakeManifest) Payload() (string, []byte, error) { return f.MediaType, []byte{}, nil } + +type FakeRegistryClientImageMetadataProvider struct { + MediaType string + Result *dockerv1client.DockerImageConfig + Manifest FakeManifest + Digest string + Ref *reference.DockerImageReference +} +type FakeManifest struct { + MediaType string +} + +func (f *FakeRegistryClientImageMetadataProvider) GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) { + _, _, err := registryclient.GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return nil, fmt.Errorf("failed to retrieve manifest %s: %w", imageRef, err) + } + return &FakeManifest{ + f.MediaType, + }, nil +} + +func (f *FakeRegistryClientImageMetadataProvider) GetDigest(ctx context.Context, imageRef string, pullSecret []byte) (digest.Digest, *reference.DockerImageReference, error) { + var err error + _, f.Ref, err = registryclient.GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return "", nil, fmt.Errorf("failed to retrieve manifest %s: %w", imageRef, err) + } + f.Ref.ID = f.Digest + return digest.Digest(f.Digest), f.Ref, nil +} + +func (f *FakeRegistryClientImageMetadataProvider) GetMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) { + return f.Result, []distribution.Descriptor{}, nil, nil +} + +func (f *FakeRegistryClientImageMetadataProvider) GetOverride(ctx context.Context, imageRef string, pullSecret []byte) (*reference.DockerImageReference, error) { + return f.Ref, nil +} diff --git a/support/util/imagemetadata.go b/support/util/imagemetadata.go index c9960b7187..b268f5a26d 100644 --- a/support/util/imagemetadata.go +++ b/support/util/imagemetadata.go @@ -12,6 +12,7 @@ import ( "github.com/golang/groupcache/lru" "k8s.io/client-go/rest" + "github.com/opencontainers/go-digest" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/releaseinfo" "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/dockerv1client" @@ -24,10 +25,16 @@ import ( var ( imageMetadataCache = lru.New(1000) + manifestsCache = lru.New(1000) + digestCache = lru.New(1000) ) type ImageMetadataProvider interface { ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) + GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) + GetDigest(ctx context.Context, imageRef string, pullSecret []byte) (digest.Digest, *reference.DockerImageReference, error) + GetMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) + GetOverride(ctx context.Context, imageRef string, pullSecret []byte) (*reference.DockerImageReference, error) } type RegistryClientImageMetadataProvider struct { @@ -44,14 +51,12 @@ type RegistryClientImageMetadataProvider struct { // no further fetching occurs. Only if both cache lookups fail, the image metadata is fetched and // stored in the cache. func (r *RegistryClientImageMetadataProvider) ImageMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, error) { - log := ctrl.LoggerFrom(ctx) var ( repo distribution.Repository ref *reference.DockerImageReference parsedImageRef reference.DockerImageReference err error - overrideFound bool ) parsedImageRef, err = reference.Parse(imageRef) @@ -62,40 +67,17 @@ func (r *RegistryClientImageMetadataProvider) ImageMetadata(ctx context.Context, // There are no ICSPs/IDMSs to process. // That means the image reference should be pulled from the external registry if len(r.OpenShiftImageRegistryOverrides) == 0 { - parsedImageRef, err = reference.Parse(imageRef) - if err != nil { - return nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) - } - // If the image reference contains a digest, immediately look it up in the cache if parsedImageRef.ID != "" { if imageConfigObject, exists := imageMetadataCache.Get(parsedImageRef.ID); exists { return imageConfigObject.(*dockerv1client.DockerImageConfig), nil } } - ref = &parsedImageRef - repo, err = getRepository(ctx, *ref, pullSecret) - if err != nil { - return nil, err - } } // Get the image repo info based the source/mirrors in the ICSPs/IDMSs - for source, mirrors := range r.OpenShiftImageRegistryOverrides { - for _, mirror := range mirrors { - ref, overrideFound, err = GetRegistryOverrides(ctx, parsedImageRef, source, mirror) - if err != nil { - log.Info(fmt.Sprintf("failed to find registry override for image reference %q with source, %s, mirror %s: %s", imageRef, source, mirror, err.Error())) - continue - } - break - } - // We found a successful source/mirror combo so break continuing any further source/mirror combos - if overrideFound { - break - } - } + ref = seekOverride(ctx, r.OpenShiftImageRegistryOverrides, parsedImageRef) // If the image reference contains a digest, immediately look it up in the cache if ref.ID != "" { @@ -131,6 +113,248 @@ func (r *RegistryClientImageMetadataProvider) ImageMetadata(ctx context.Context, return config, nil } +// GetOverride returns the image reference override based on the source/mirrors in the ICSPs/IDMSs +func (r *RegistryClientImageMetadataProvider) GetOverride(ctx context.Context, imageRef string, pullSecret []byte) (*reference.DockerImageReference, error) { + + var ( + ref *reference.DockerImageReference + parsedImageRef reference.DockerImageReference + err error + ) + + parsedImageRef, err = reference.Parse(imageRef) + if err != nil { + return nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + + // There are no ICSPs/IDMSs to process. + // That means the image reference should be pulled from the external registry + if len(r.OpenShiftImageRegistryOverrides) == 0 { + ref = &parsedImageRef + } + + ref = seekOverride(ctx, r.OpenShiftImageRegistryOverrides, parsedImageRef) + + return ref, nil +} + +func (r *RegistryClientImageMetadataProvider) GetDigest(ctx context.Context, imageRef string, pullSecret []byte) (digest.Digest, *reference.DockerImageReference, error) { + + var ( + repo distribution.Repository + ref *reference.DockerImageReference + parsedImageRef reference.DockerImageReference + err error + srcDigest digest.Digest + ) + + parsedImageRef, err = reference.Parse(imageRef) + if err != nil { + return "", nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + + // There are no ICSPs/IDMSs to process. + // That means the image reference should be pulled from the external registry + if len(r.OpenShiftImageRegistryOverrides) == 0 { + // If the image name is in the cache, return early + if imageDigest, exists := digestCache.Get(imageRef); exists { + parsedImageRef.ID = string(imageDigest.(digest.Digest)) + return imageDigest.(digest.Digest), &parsedImageRef, nil + } + + ref = &parsedImageRef + } + + // Get the image repo info based the source/mirrors in the ICSPs/IDMSs + ref = seekOverride(ctx, r.OpenShiftImageRegistryOverrides, parsedImageRef) + + composedRef := fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.NameString()) + + // If the overriden image name is in the cache, return early + if imageDigest, exists := digestCache.Get(composedRef); exists { + ref.ID = string(imageDigest.(digest.Digest)) + return imageDigest.(digest.Digest), ref, nil + } + + repo, composedParsedRef, err := GetRepoSetup(ctx, composedRef, pullSecret) + if err != nil || repo == nil { + return "", nil, fmt.Errorf("failed to create repository client for %s: %w", ref.DockerClientDefaults().RegistryURL(), err) + } + + switch { + case len(composedParsedRef.ID) > 0: + srcDigest = digest.Digest(composedParsedRef.ID) + + case len(composedParsedRef.Tag) > 0: + desc, err := repo.Tags(ctx).Get(ctx, composedParsedRef.Tag) + if err != nil { + return "", nil, err + } + srcDigest = desc.Digest + composedParsedRef.ID = string(srcDigest) + } + + digestCache.Add(composedRef, srcDigest) + digestCache.Add(imageRef, srcDigest) + + return srcDigest, composedParsedRef, nil +} + +// GetManifest returns the manifest for a given image using the given pull secret +// to authenticate. This lookup uses a cache based on the image digest. If The +// reference of the image contains a digest (which is the mainline case for images in a release payload), +// the digest is parsed from the image reference and then used to lookup the manifest in the +// cache and return it with the ImageOverrides already included. +func (r *RegistryClientImageMetadataProvider) GetManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, error) { + + var ( + ref *reference.DockerImageReference + parsedImageRef reference.DockerImageReference + err error + srcDigest digest.Digest + ) + + parsedImageRef, err = reference.Parse(imageRef) + if err != nil { + return nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + + // There are no ICSPs/IDMSs to process. + // That means the image reference should be pulled from the external registry + if len(r.OpenShiftImageRegistryOverrides) == 0 { + // If the image reference contains a digest, immediately look it up in the cache + if parsedImageRef.ID != "" { + if manifest, exists := manifestsCache.Get(parsedImageRef.ID); exists { + return manifest.(distribution.Manifest), nil + } + } + ref = &parsedImageRef + } + + // Get the image repo info based the source/mirrors in the ICSPs/IDMSs + ref = seekOverride(ctx, r.OpenShiftImageRegistryOverrides, parsedImageRef) + + // If the image reference contains a digest, immediately look it up in the cache + if ref.ID != "" { + if manifest, exists := manifestsCache.Get(ref.ID); exists { + return manifest.(distribution.Manifest), nil + } + } + + composedRef := fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.NameString()) + + digestsManifest, srcDigest, err := getManifest(ctx, composedRef, pullSecret) + if err != nil { + return nil, err + } + manifestsCache.Add(srcDigest, digestsManifest) + + return digestsManifest, nil +} + +func (r *RegistryClientImageMetadataProvider) GetMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) { + + var ( + ref *reference.DockerImageReference + parsedImageRef reference.DockerImageReference + err error + ) + + if len(r.OpenShiftImageRegistryOverrides) == 0 { + return getMetadata(ctx, imageRef, pullSecret) + } + + parsedImageRef, err = reference.Parse(imageRef) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + + // Get the image repo info based the source/mirrors in the ICSPs/IDMSs + ref = seekOverride(ctx, r.OpenShiftImageRegistryOverrides, parsedImageRef) + composedRef := fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.NameString()) + + return getMetadata(ctx, composedRef, pullSecret) +} + +// getManifest gets the manifest from an image +func getManifest(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Manifest, digest.Digest, error) { + repo, ref, err := GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return nil, "", err + } + + var srcDigest digest.Digest + if len(ref.Tag) > 0 { + desc, err := repo.Tags(ctx).Get(ctx, ref.Tag) + if err != nil { + return nil, "", err + } + srcDigest = desc.Digest + } + + if len(ref.ID) > 0 { + srcDigest = digest.Digest(ref.ID) + } + + manifests, err := repo.Manifests(ctx) + if err != nil { + return nil, "", err + } + + digestsManifest, err := manifests.Get(ctx, srcDigest, manifest.PreferManifestList) + if err != nil { + return nil, "", err + } + + return digestsManifest, srcDigest, nil +} + +func getMetadata(ctx context.Context, imageRef string, pullSecret []byte) (*dockerv1client.DockerImageConfig, []distribution.Descriptor, distribution.BlobStore, error) { + repo, ref, err := GetRepoSetup(ctx, imageRef, pullSecret) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get repo setup: %w", err) + } + firstManifest, location, err := manifest.FirstManifest(ctx, *ref, repo) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to obtain root manifest for %s: %w", imageRef, err) + } + imageConfig, layers, err := manifest.ManifestToImageConfig(ctx, firstManifest, repo.Blobs(ctx), location) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to obtain image layers for %s: %w", imageRef, err) + } + return imageConfig, layers, repo.Blobs(ctx), nil +} + +// GetRepoSetup connects to a repo and pulls the imageRef's docker image information from the repo. Returns the repo and the docker image. +func GetRepoSetup(ctx context.Context, imageRef string, pullSecret []byte) (distribution.Repository, *reference.DockerImageReference, error) { + var dockerImageRef *reference.DockerImageReference + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to create secure transport: %w", err) + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return nil, nil, fmt.Errorf("failed to create insecure transport: %w", err) + } + credStore, err := dockercredentials.NewFromBytes(pullSecret) + if err != nil { + return nil, nil, fmt.Errorf("GetRepoSetup - failed to parse docker credentials: %w", err) + } + registryContext := registryclient.NewContext(rt, insecureRT).WithCredentials(credStore). + WithRequestModifiers(transport.NewHeaderRequestModifier(http.Header{http.CanonicalHeaderKey("User-Agent"): []string{rest.DefaultKubernetesUserAgent()}})) + + ref, err := reference.Parse(imageRef) + dockerImageRef = &ref + if err != nil { + return nil, nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + repo, err := registryContext.Repository(ctx, ref.DockerClientDefaults().RegistryURL(), ref.RepositoryName(), false) + if err != nil { + return nil, nil, fmt.Errorf("failed to create repository client for %s: %w", ref.DockerClientDefaults().RegistryURL(), err) + } + return repo, dockerImageRef, nil +} + func getRepository(ctx context.Context, ref reference.DockerImageReference, pullSecret []byte) (distribution.Repository, error) { credStore, err := dockercredentials.NewFromBytes(pullSecret) if err != nil { @@ -170,13 +394,33 @@ func GetRegistryOverrides(ctx context.Context, ref reference.DockerImageReferenc return nil, false, fmt.Errorf("failed to parse source image reference %q: %w", source, err) } - if sourceRef.Namespace == ref.Namespace && sourceRef.Name == ref.Name { - log.Info("registry override coincidence found", "original", fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.Name), "mirror", mirror) - mirrorRef, err := reference.Parse(mirror) + mirrorRef, err := reference.Parse(mirror) + if err != nil { + return nil, false, fmt.Errorf("failed to parse mirror image reference %q: %w", source, err) + } + + // docker lib, by default will empty the Namespace once we pass an override with just a Namespace + // and it will asume that is the Name instead + if sourceRef.Namespace == "" { + if sourceRef.Name == ref.Namespace { + composedImage := fmt.Sprintf("%s/%s/%s", mirrorRef.Registry, ref.Namespace, ref.NameString()) + composedRef, err := reference.Parse(composedImage) + if err != nil { + return nil, false, fmt.Errorf("failed to parse composed image reference (partial match) %q: %w", source, err) + } + log.Info("registry override coincidence found (namespace)", "original", fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.NameString()), "mirror", mirror, "composed", composedRef) + return &composedRef, true, nil + } + } + + if ref.Namespace == sourceRef.Namespace && ref.Name == sourceRef.Name { + composedImage := fmt.Sprintf("%s/%s/%s", mirrorRef.Registry, mirrorRef.Namespace, ref.NameString()) + composedRef, err := reference.Parse(composedImage) if err != nil { - return nil, false, fmt.Errorf("failed to parse mirror image reference %q: %w", mirrorRef.Name, err) + return nil, false, fmt.Errorf("failed to parse composed image reference (exact match) %q: %w", source, err) } - return &mirrorRef, true, nil + log.Info("registry override coincidence found (exact match)", "original", fmt.Sprintf("%s/%s/%s", ref.Registry, ref.Namespace, ref.NameString()), "mirror", mirror, "composed", composedRef) + return &composedRef, true, nil } return &ref, false, nil @@ -207,3 +451,20 @@ func GetPayloadVersion(ctx context.Context, releaseImageProvider releaseinfo.Pro } return &version, nil } + +func seekOverride(ctx context.Context, openshiftImageRegistryOverrides map[string][]string, parsedImageReference reference.DockerImageReference) *reference.DockerImageReference { + log := ctrl.LoggerFrom(ctx) + for source, mirrors := range openshiftImageRegistryOverrides { + for _, mirror := range mirrors { + ref, overrideFound, err := GetRegistryOverrides(context.Background(), parsedImageReference, source, mirror) + if err != nil { + log.Info(fmt.Sprintf("failed to find registry override for image reference %q with source, %s, mirror %s: %s", parsedImageReference, source, mirror, err.Error())) + continue + } + if overrideFound { + return ref + } + } + } + return &parsedImageReference +} diff --git a/support/util/imagemetadata_test.go b/support/util/imagemetadata_test.go index 1b54521287..9f04144f29 100644 --- a/support/util/imagemetadata_test.go +++ b/support/util/imagemetadata_test.go @@ -5,6 +5,7 @@ import ( "testing" . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/reference" ) @@ -82,3 +83,255 @@ func TestGetRegistryOverrides(t *testing.T) { }) } } + +func TestGetManifest(t *testing.T) { + ctx := context.TODO() + pullSecret := []byte("{}") + + testsCases := []struct { + name string + imageRef string + pullSecret []byte + expectedErr bool + validateCache bool + expectedDigest digest.Digest + }{ + { + name: "if failed to parse image reference", + imageRef: "invalid-image-ref", + pullSecret: pullSecret, + expectedErr: true, + }, + { + name: "Pull x86 manifest", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-x86_64", + pullSecret: pullSecret, + expectedErr: false, + }, + { + name: "Pull x86 manifest from cache", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-x86_64", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:2a50e5d5267916078145731db740bbc85ee764e1a194715fd986ab5bf9a3414e", + }, + { + name: "Pull Multiarch manifest", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-multi", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:727276732f03d8d5a2374efa3d01fb0ed9f65b32488b862e9a9d2ff4cde89ff6", + }, + } + + for _, tc := range testsCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + provider := &RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: map[string][]string{}, + } + + manifest, err := provider.GetManifest(ctx, tc.imageRef, tc.pullSecret) + if tc.expectedErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(manifest).NotTo(BeNil()) + } + + if tc.validateCache { + _, exists := manifestsCache.Get(tc.expectedDigest) + g.Expect(exists).To(BeTrue()) + } + }) + } +} +func TestGetDigest(t *testing.T) { + ctx := context.TODO() + pullSecret := []byte("{}") + + testsCases := []struct { + name string + imageRef string + pullSecret []byte + expectedErr bool + validateCache bool + expectedDigest digest.Digest + }{ + { + name: "if failed to parse image reference", + imageRef: "invalid-image-ref", + pullSecret: pullSecret, + expectedErr: true, + }, + { + name: "Multiaarch image digest", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-multi", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:727276732f03d8d5a2374efa3d01fb0ed9f65b32488b862e9a9d2ff4cde89ff6", + }, + { + name: "amd64 Image digest is found in cache", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-x86_64", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:2a50e5d5267916078145731db740bbc85ee764e1a194715fd986ab5bf9a3414e", + }, + { + name: "amd64 Image digest, recover from cache", + imageRef: "quay.io/openshift-release-dev/ocp-release:4.16.12-x86_64", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:2a50e5d5267916078145731db740bbc85ee764e1a194715fd986ab5bf9a3414e", + }, + { + name: "Image with digest and not tag", + imageRef: "quay.io/openshift-release-dev/ocp-release@sha256:e96047c50caf0aaffeaf7ed0fe50bd3f574ad347cd0f588a56b876f79cc29d3e", + pullSecret: pullSecret, + expectedErr: false, + validateCache: true, + expectedDigest: "sha256:e96047c50caf0aaffeaf7ed0fe50bd3f574ad347cd0f588a56b876f79cc29d3e", + }, + } + + for _, tc := range testsCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + provider := &RegistryClientImageMetadataProvider{ + OpenShiftImageRegistryOverrides: map[string][]string{}, + } + + digest, ref, err := provider.GetDigest(ctx, tc.imageRef, tc.pullSecret) + if tc.expectedErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(digest).To(Equal(tc.expectedDigest)) + g.Expect(ref).NotTo(BeNil()) + } + + if tc.validateCache { + _, exists := digestCache.Get(tc.imageRef) + g.Expect(exists).To(BeTrue()) + } + }) + } +} +func TestSeekOverride(t *testing.T) { + testsCases := []struct { + name string + overrides map[string][]string + imageRef reference.DockerImageReference + expectedImgRef *reference.DockerImageReference + }{ + { + name: "if no overrides are provided", + overrides: map[string][]string{}, + imageRef: reference.DockerImageReference{ + Registry: "quay.io", + Name: "ocp-release", + Namespace: "openshift-release-dev", + Tag: "4.15.0-rc.0-multi", + }, + expectedImgRef: &reference.DockerImageReference{ + Registry: "quay.io", + Name: "ocp-release", + Namespace: "openshift-release-dev", + Tag: "4.15.0-rc.0-multi", + }, + }, + { + name: "if registry override exact coincidence is found", + overrides: fakeOverrides(), + imageRef: reference.DockerImageReference{ + Registry: "quay.io", + Name: "ocp-release", + Namespace: "openshift-release-dev", + Tag: "4.15.0-rc.0-multi", + }, + expectedImgRef: &reference.DockerImageReference{ + Registry: "myregistry1.io", + Name: "ocp-release", + Namespace: "openshift-release-dev", + Tag: "4.15.0-rc.0-multi", + }, + }, + { + name: "if registry override partial coincidence is found", + overrides: fakeOverrides(), + imageRef: reference.DockerImageReference{ + Registry: "quay.io", + Name: "mce-image", + Namespace: "mce", + Tag: "latest", + }, + expectedImgRef: &reference.DockerImageReference{ + Registry: "myregistry.io", + Name: "mce-image", + Namespace: "mce", + Tag: "latest", + }, + }, + { + name: "if registry override coincidence is not found", + overrides: fakeOverrides(), + imageRef: reference.DockerImageReference{ + Registry: "quay.io", + Name: "testimage", + Namespace: "test-namespace", + Tag: "latest", + }, + expectedImgRef: &reference.DockerImageReference{ + Registry: "quay.io", + Name: "testimage", + Namespace: "test-namespace", + Tag: "latest", + }, + }, + { + name: "if failed to find registry override", + overrides: fakeOverrides(), + imageRef: reference.DockerImageReference{ + Registry: "quay.io", + Name: "cnv-image", + Namespace: "cnv", + Tag: "latest", + }, + expectedImgRef: &reference.DockerImageReference{ + Registry: "quay.io", + Name: "cnv-image", + Namespace: "cnv", + Tag: "latest", + }, + }, + } + + for _, tc := range testsCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.TODO() + g := NewGomegaWithT(t) + imgRef := seekOverride(ctx, tc.overrides, tc.imageRef) + g.Expect(imgRef).To(Equal(tc.expectedImgRef)) + }) + } +} + +func fakeOverrides() map[string][]string { + return map[string][]string{ + "quay.io/openshift-release-dev/ocp-release": { + "myregistry1.io/openshift-release-dev/ocp-release", + "myregistry2.io/openshift-release-dev/ocp-release", + }, + "quay.io/mce": { + "myregistry.io/mce", + }, + } +} diff --git a/support/util/util.go b/support/util/util.go index 2a9e8567c7..0a8feabd83 100644 --- a/support/util/util.go +++ b/support/util/util.go @@ -398,7 +398,7 @@ func DetermineHostedClusterPayloadArch(ctx context.Context, c client.Client, hc return "", fmt.Errorf("expected %s key in pull secret", corev1.DockerConfigJsonKey) } - isMultiArchReleaseImage, err := registryclient.IsMultiArchManifestList(ctx, hc.Spec.Release.Image, pullSecretBytes) + isMultiArchReleaseImage, err := registryclient.IsMultiArchManifestList(ctx, hc.Spec.Release.Image, pullSecretBytes, imageMetadataProvider) if err != nil { return "", fmt.Errorf("failed to determine if release image multi-arch: %w", err) }