diff --git a/docs/dev/object-deletion.md b/docs/dev/object-deletion.md new file mode 100644 index 000000000..d48c3bdb8 --- /dev/null +++ b/docs/dev/object-deletion.md @@ -0,0 +1,110 @@ +# Manifest Annotation For Object Deletion + +Developers can remove any of the currently managed CVO objects by modifying an existing manifest and adding the delete annotation `.metadata.annotations["release.openshift.io/delete"]="true"`. This manifest annotation is a request for the CVO to delete the in-cluster object instead of creating/updating it. + +Actual object deletion and subsequent deletion monitoring and status checking will only occur during an upgrade. During initial installation the delete annotation prevents further processing of the manifest since the given object should not be created. + +## Implementation Details + +When the following annotation appears in a CVO supported manifest and is set to `true` the associated object will be removed from the cluster by the CVO. +Values other than `true` will result in a CVO failure and should therefore result in CVO CI failure. +```yaml +apiVersion: apps/v1 +... +metadata: +... + annotations: + release.openshift.io/delete: "true" +``` +The existing CVO ordering scheme defined [here](operators.md) is also used for object removal. This provides a simple and familiar method of deleting multiple objects by reversing the order in which they were created. It is the developer's responsibility to ensure proper deletion ordering and to ensure that all items originally created by an object are deleted when that object is deleted. For example, an operator may have to be modified, or a new operator created, to take explicit actions to remove external resources. The modified or new operator would then be removed in a subsequent update. + +Similar to how CVO handles create/update requests, deletion requests are implemented in a non-blocking manner whereby CVO issues the initial request to delete an object kicking off resource finalization and after which resource removal. CVO does not wait for actual resource removal but instead continues. CVO logs when a delete is initiated, that the delete is ongoing when a manifest is processed again and found to have a deletion time stamp, and delete completion upon resource finalization. + +If an object cannot be successfully removed CVO will set `Upgradeable=False` which in turn blocks cluster update to the next minor release. + +## Examples + +The following examples provide guidance to OpenShift developers on how resources may be removed but this will vary depending on the component. +Ultimately it is the developer's responsibility to ensure the removal works by thoroughly testing. +In all cases, and as general guidance, an operator should never allow itself to be removed if the operator's operand has not been removed. + +### The autoscaler operator + +Remove the cluster-autoscaler-operator deployment. +The existing cluster-autoscaler-operator deployment manifest 0000_50_cluster-autoscaler-operator_07_deployment.yaml is modified to contain the delete annotation: +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler-operator + namespace: openshift-machine-api + annotations: + release.openshift.io/delete: "true" +... +``` +Additional manifest properties such as `spec` may be set if convenient (e.g. because you are looking to make a minimal change vs. a previous version of the manifest), but those properties have no affect on manifests with the delete annotation. + +### The service-catalog operators + +In release 4.5 two jobs were created to remove the Service Catalog - openshift-service-catalog-controller-manager-remover and openshift-service-catalog-apiserver-remover. +In release 4.6, these jobs and all their supporting cluster objects also needed to be removed. +The following example shows how to do Service Catalog removal using the object deletion manifest annotation. + +The Service Catalog is composed of two components, the cluster-svcat-apiserver-operator and the cluster-svcat-controller-manager-operator. +Each of these components use manifests for creation/update of the component's required resources: namespace, roles, operator deployment, etc. +The cluster-svcat-apiserver-operator had [these associated manifests][svcat-apiserver-4.4-manifests]. + +The deletion annotation would be added to these manifests: + +* `0000_50_cluster-svcat-apiserver-operator_00_namespace.yaml` containing the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_50_cluster-svcat-apiserver-operator_02_config.crd.yaml` containing a cluster-scoped CRD. +* `0000_50_cluster-svcat-apiserver-operator_03_config.cr.yaml` containing a cluster-scoped, create-only ServiceCatalogAPIServer. +* `0000_50_cluster-svcat-apiserver-operator_04_roles.yaml` containing a cluster-scoped ClusterRoleBinding. +* `0000_50_cluster-svcat-apiserver-operator_08_cluster-operator.yaml` containing a cluster-scoped ClusterOperator. + +These manifests would be dropped because their removal would occur as part of one of the above resource deletions: + +* `0000_50_cluster-svcat-apiserver-operator_03_configmap.yaml` containing a ConfigMap in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_50_cluster-svcat-apiserver-operator_03_version-configmap.yaml` containing another ConfigMap in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_50_cluster-svcat-apiserver-operator_05_serviceaccount.yaml` containing a ServiceAccount in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_50_cluster-svcat-apiserver-operator_06_service.yaml` containing a Service in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_50_cluster-svcat-apiserver-operator_07_deployment.yaml` containing a Deployment in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_90_cluster-svcat-apiserver-operator_00_prometheusrole.yaml` containing a Role in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_90_cluster-svcat-apiserver-operator_01_prometheusrolebinding.yaml` containing a RoleBinding in the `openshift-service-catalog-apiserver-operator` namespace. +* `0000_90_cluster-svcat-apiserver-operator_02-operator-servicemonitor.yaml` containing a ServiceMonitor in the `openshift-service-catalog-apiserver-operator` namespace. + +So the remaining manifests with deletion annotations would be the namespace and the cluster-scoped CRD, ServiceCatalogAPIServer, ClusterRoleBinding, and ClusterOperator. +The ordering of the surviving manifests would not be particularly important, although keeping the namespace first would avoid removing the ClusterRoleBinding while the consuming Deployment was still running. +If multiple deletions are required, it is up to the developer to name the manifests such that deletions occur in the correct order. + +Similar handling would be required for the svcat-controller-manager operator. + +If resources external to kubernetes must be removed the developer must provide the means to do so. +This is expected to be done through modification of an operator to do the removals during it's finalization. +If operator modification for object removal is necessary that operator would be deleted in a subsequent update. + +The deletion manifests described above would have been preserved through 4.5.z release and removed in 4.6. +See the [Subsequent Releases Strategy](#subsequent-releases-strategy) section for details. + +## Removing functionality that users might notice + +Below is the flow for removing functionality that users might notice, like the web console. + +* The first step is deprecating the functionality. + During the deprecation release 4.y, the functionality should remain available, with the operator setting Upgradeable=False and linking release notes like [these][deprecated-marketplace-apis]. +* Cluster Administrators must follow the linked release notes to opt in to the removal before updating to the next minor release in 4.(y+1). + When the administrator opts in to the removal, the operator should stop setting Upgradeable=False. +* Depending on how the engineering team that owns the functionality implements its removal, the operand components may be removed when Cluster Administrators opt-in to the removal, or they may be left running and be removed during the transition to the next minor release 4.(y+1). +* During the update to the next minor release 4.(y+1) the manifest delete annotation would be used to remove the operator, and, if they have not already been removed as part of the opt-in in release 4.y, any remaining operand components. + +## Subsequent Releases Strategy + +Special consideration must be given to subsequent updates which may still contain manifests with the delete annotation. +These manifests will result in `object no longer exists` errors assuming the current release had properly and fully removed the given objects. +It is acceptable for subsequent z-level updates to still contain the delete manifests but minor level updates should not and therefore the handling of the delete error will differ between these update levels. +A z-level update will be allowed to proceed while a minor level update will be blocked. +This will be accomplished through the existing CVO precondition mechanism which already behaves in this manner with regard to z-level and minor updates. + +[svcat-apiserver-4.4-manifests]: https://github.com/openshift/cluster-svcat-apiserver-operator/tree/aa7927fbfe8bf165c5b84167b7c3f5d9cb394e14/manifests +[deprecated-marketplace-apis]: https://docs.openshift.com/container-platform/4.4/release_notes/ocp-4-4-release-notes.html#ocp-4-4-marketplace-apis-deprecated diff --git a/docs/dev/operators.md b/docs/dev/operators.md index c03e46a83..addc06bf9 100644 --- a/docs/dev/operators.md +++ b/docs/dev/operators.md @@ -30,12 +30,12 @@ components that have the same run level - for instance, `0000_70_cluster-monitor preserving the order of tasks within the component. Ordering is only applied during upgrades, where some components rely on another component -being updated first. As a convenience, the CVO guarantees that components at an earlier -run level will be created or updated before your component is invoked. Note however that -components without `ClusterOperator` objects defined may not be fully deployed when your -component is executed, so always ensure your prerequisites know that they must correctly -obey the `ClusterOperator` protocol to be available. More sophisticated components should -observe the prerequisite `ClusterOperator`s directly and use the `versions` field to +being updated or deleted first. As a convenience, the CVO guarantees that components at an +earlier run level will be created, updated, or deleted before your component is invoked. Note +however that components without `ClusterOperator` objects defined may not be fully deployed +when your component is executed, so always ensure your prerequisites know that they must +correctly obey the `ClusterOperator` protocol to be available. More sophisticated components +should observe the prerequisite `ClusterOperator`s directly and use the `versions` field to enforce safety. ## How do I get added to the release image? @@ -68,6 +68,8 @@ You need the following: In your deployment you can reference the latest development version of your operator image (quay.io/openshift/origin-machine-api-operator:latest). If you have other hard-coded image strings, try to put them as environment variables on your deployment or as a config map. +Manifest files may also be used to delete your object [more info here](object-deletion.md). + ### Names of manifest files Your manifests will be applied in alphabetical order by the CVO, so name your files in the order you want them run. diff --git a/lib/resourcebuilder/resourcebuilder.go b/lib/resourcebuilder/resourcebuilder.go index bc5a4e5d2..342328546 100644 --- a/lib/resourcebuilder/resourcebuilder.go +++ b/lib/resourcebuilder/resourcebuilder.go @@ -13,6 +13,7 @@ import ( imageclientv1 "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" securityclientv1 "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" "github.com/openshift/cluster-version-operator/lib/resourceapply" + "github.com/openshift/cluster-version-operator/lib/resourcedelete" "github.com/openshift/cluster-version-operator/lib/resourceread" "github.com/openshift/library-go/pkg/manifest" appsv1 "k8s.io/api/apps/v1" @@ -74,6 +75,7 @@ func (b *builder) WithModifier(f MetaV1ObjectModifierFunc) Interface { func (b *builder) Do(ctx context.Context) error { obj := resourceread.ReadOrDie(b.raw) + updatingMode := (b.mode == UpdatingMode) switch typedObject := obj.(type) { case *imagev1.ImageStream: @@ -87,101 +89,166 @@ func (b *builder) Do(ctx context.Context) error { if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplySecurityContextConstraintsv1(ctx, b.securityClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteSecurityContextConstraintsv1(ctx, b.securityClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplySecurityContextConstraintsv1(ctx, b.securityClientv1, typedObject); err != nil { + return err + } } case *appsv1.DaemonSet: if b.modifier != nil { b.modifier(typedObject) } - if err := b.modifyDaemonSet(ctx, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteDaemonSetv1(ctx, b.appsClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if err := b.modifyDaemonSet(ctx, typedObject); err != nil { + return err + } + if _, _, err := resourceapply.ApplyDaemonSetv1(ctx, b.appsClientv1, typedObject); err != nil { + return err + } + return b.checkDaemonSetHealth(ctx, typedObject) } - if _, _, err := resourceapply.ApplyDaemonSetv1(ctx, b.appsClientv1, typedObject); err != nil { - return err - } - return b.checkDaemonSetHealth(ctx, typedObject) case *appsv1.Deployment: if b.modifier != nil { b.modifier(typedObject) } - if err := b.modifyDeployment(ctx, typedObject); err != nil { - return err - } - if _, _, err := resourceapply.ApplyDeploymentv1(ctx, b.appsClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteDeploymentv1(ctx, b.appsClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if err := b.modifyDeployment(ctx, typedObject); err != nil { + return err + } + if _, _, err := resourceapply.ApplyDeploymentv1(ctx, b.appsClientv1, typedObject); err != nil { + return err + } + return b.checkDeploymentHealth(ctx, typedObject) } - return b.checkDeploymentHealth(ctx, typedObject) case *batchv1.Job: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyJobv1(ctx, b.batchClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteJobv1(ctx, b.batchClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyJobv1(ctx, b.batchClientv1, typedObject); err != nil { + return err + } + return b.checkJobHealth(ctx, typedObject) } - return b.checkJobHealth(ctx, typedObject) case *corev1.ConfigMap: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyConfigMapv1(ctx, b.coreClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteConfigMapv1(ctx, b.coreClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyConfigMapv1(ctx, b.coreClientv1, typedObject); err != nil { + return err + } } case *corev1.Namespace: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyNamespacev1(ctx, b.coreClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteNamespacev1(ctx, b.coreClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyNamespacev1(ctx, b.coreClientv1, typedObject); err != nil { + return err + } } case *corev1.Service: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyServicev1(ctx, b.coreClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteServicev1(ctx, b.coreClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyServicev1(ctx, b.coreClientv1, typedObject); err != nil { + return err + } } case *corev1.ServiceAccount: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyServiceAccountv1(ctx, b.coreClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteServiceAccountv1(ctx, b.coreClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyServiceAccountv1(ctx, b.coreClientv1, typedObject); err != nil { + return err + } } case *rbacv1.ClusterRole: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyClusterRolev1(ctx, b.rbacClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteClusterRolev1(ctx, b.rbacClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyClusterRolev1(ctx, b.rbacClientv1, typedObject); err != nil { + return err + } } case *rbacv1.ClusterRoleBinding: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyClusterRoleBindingv1(ctx, b.rbacClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteClusterRoleBindingv1(ctx, b.rbacClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyClusterRoleBindingv1(ctx, b.rbacClientv1, typedObject); err != nil { + return err + } } case *rbacv1.Role: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyRolev1(ctx, b.rbacClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteRolev1(ctx, b.rbacClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyRolev1(ctx, b.rbacClientv1, typedObject); err != nil { + return err + } } case *rbacv1.RoleBinding: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyRoleBindingv1(ctx, b.rbacClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteRoleBindingv1(ctx, b.rbacClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyRoleBindingv1(ctx, b.rbacClientv1, typedObject); err != nil { + return err + } } case *apiextensionsv1.CustomResourceDefinition: if b.modifier != nil { b.modifier(typedObject) } - if _, _, err := resourceapply.ApplyCustomResourceDefinitionv1(ctx, b.apiextensionsClientv1, typedObject); err != nil { + if deleteReq, err := resourcedelete.DeleteCustomResourceDefinitionv1(ctx, b.apiextensionsClientv1, typedObject, + updatingMode); err != nil { return err + } else if !deleteReq { + if _, _, err := resourceapply.ApplyCustomResourceDefinitionv1(ctx, b.apiextensionsClientv1, typedObject); err != nil { + return err + } } default: return fmt.Errorf("unrecognized manifest type: %T", obj) diff --git a/lib/resourcedelete/apiext.go b/lib/resourcedelete/apiext.go new file mode 100644 index 000000000..90fa54488 --- /dev/null +++ b/lib/resourcedelete/apiext.go @@ -0,0 +1,41 @@ +package resourcedelete + +import ( + "context" + "fmt" + + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeleteCustomResourceDefinitionv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteCustomResourceDefinitionv1(ctx context.Context, client apiextclientv1.CustomResourceDefinitionsGetter, + required *apiextv1.CustomResourceDefinition, updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "customresourcedefinition", + Namespace: "", + Name: required.Name, + } + existing, err := client.CustomResourceDefinitions().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.CustomResourceDefinitions().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/apireg.go b/lib/resourcedelete/apireg.go new file mode 100644 index 000000000..7b7e7fc9e --- /dev/null +++ b/lib/resourcedelete/apireg.go @@ -0,0 +1,41 @@ +package resourcedelete + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + apiregclientv1 "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" +) + +// DeleteAPIServicev1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteAPIServicev1(ctx context.Context, client apiregclientv1.APIServicesGetter, required *apiregv1.APIService, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "apiservice", + Namespace: "", + Name: required.Name, + } + existing, err := client.APIServices().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.APIServices().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/apps.go b/lib/resourcedelete/apps.go new file mode 100644 index 000000000..3653ec4d2 --- /dev/null +++ b/lib/resourcedelete/apps.go @@ -0,0 +1,72 @@ +package resourcedelete + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1" +) + +// DeleteDeploymentv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteDeploymentv1(ctx context.Context, client appsclientv1.DeploymentsGetter, required *appsv1.Deployment, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "deployment", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.Deployments(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Deployments(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteDaemonSetv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteDaemonSetv1(ctx context.Context, client appsclientv1.DaemonSetsGetter, required *appsv1.DaemonSet, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "daemonset", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.DaemonSets(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.DaemonSets(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/batch.go b/lib/resourcedelete/batch.go new file mode 100644 index 000000000..0580efc80 --- /dev/null +++ b/lib/resourcedelete/batch.go @@ -0,0 +1,41 @@ +package resourcedelete + +import ( + "context" + "fmt" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + batchclientv1 "k8s.io/client-go/kubernetes/typed/batch/v1" +) + +// DeleteJobv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteJobv1(ctx context.Context, client batchclientv1.JobsGetter, required *batchv1.Job, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "job", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.Jobs(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Jobs(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/core.go b/lib/resourcedelete/core.go new file mode 100644 index 000000000..f01544241 --- /dev/null +++ b/lib/resourcedelete/core.go @@ -0,0 +1,134 @@ +package resourcedelete + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// DeleteNamespacev1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteNamespacev1(ctx context.Context, client coreclientv1.NamespacesGetter, required *corev1.Namespace, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "namespace", + Namespace: "", + Name: required.Name, + } + existing, err := client.Namespaces().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Namespaces().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteServicev1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteServicev1(ctx context.Context, client coreclientv1.ServicesGetter, required *corev1.Service, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "service", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.Services(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Services(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteServiceAccountv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteServiceAccountv1(ctx context.Context, client coreclientv1.ServiceAccountsGetter, required *corev1.ServiceAccount, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "serviceaccount", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.ServiceAccounts(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.ServiceAccounts(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteConfigMapv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteConfigMapv1(ctx context.Context, client coreclientv1.ConfigMapsGetter, required *corev1.ConfigMap, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "configmap", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.ConfigMaps(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.ConfigMaps(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/helper.go b/lib/resourcedelete/helper.go new file mode 100644 index 000000000..db1a1b186 --- /dev/null +++ b/lib/resourcedelete/helper.go @@ -0,0 +1,145 @@ +package resourcedelete + +import ( + "fmt" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +const DeleteAnnotation = "release.openshift.io/delete" + +type Resource struct { + Kind string + Namespace string + Name string +} + +type deleteTimes struct { + Requested time.Time + Expected *metav1.Time + Verified time.Time +} + +var ( + deletedResources = struct { + lock sync.RWMutex + m map[Resource]deleteTimes + }{m: make(map[Resource]deleteTimes)} +) + +// ValidDeleteAnnotation returns whether the delete annotation is found and an error if it is found +// but not set to "true". +func ValidDeleteAnnotation(annotations map[string]string) (bool, error) { + if value, ok := annotations[DeleteAnnotation]; !ok { + return false, nil + } else if value != "true" { + return true, fmt.Errorf("Invalid delete annotation \"%s\" value: \"%s\"", DeleteAnnotation, value) + } + return true, nil +} + +func (r Resource) uniqueName() string { + if len(r.Namespace) == 0 { + return r.Name + } + return r.Namespace + "/" + r.Name +} + +func (r Resource) String() string { + return fmt.Sprintf("%s \"%s\"", r.Kind, r.uniqueName()) +} + +// SetDeleteRequested creates or updates entry in map to indicate resource deletion has been requested. +func SetDeleteRequested(obj metav1.Object, resource Resource) { + times := deleteTimes{ + Requested: time.Now(), + Expected: obj.GetDeletionTimestamp(), + } + deletedResources.lock.Lock() + deletedResources.m[resource] = times + deletedResources.lock.Unlock() + klog.V(4).Infof("Delete requested for %s.", resource) +} + +// SetDeleteVerified updates map entry to indicate resource deletion has been completed. +func SetDeleteVerified(resource Resource) { + times := deleteTimes{ + Verified: time.Now(), + } + deletedResources.lock.Lock() + deletedResources.m[resource] = times + deletedResources.lock.Unlock() + klog.V(4).Infof("Delete of %s completed.", resource) +} + +// getDeleteTimes returns map entry for given resource. +func getDeleteTimes(resource Resource) (deleteTimes, bool) { + deletedResources.lock.Lock() + defer deletedResources.lock.Unlock() + deletionTimes, ok := deletedResources.m[resource] + return deletionTimes, ok +} + +// setDeleteRequestedAndVerified creates or updates a map entry to indicate resource deletion has been requested +// and completed. +func setDeleteRequestedAndVerified(resource Resource) { + times := deleteTimes{ + Requested: time.Now(), + Verified: time.Now(), + } + deletedResources.lock.Lock() + deletedResources.m[resource] = times + deletedResources.lock.Unlock() + klog.Warningf("%s has already been removed.", resource) +} + +// GetDeleteProgress checks if resource deletion has been requested. If it has it checks if the deletion has completed +// and if not logs deletion progress. This method returns an indication of whether resource deletion has already been +// requested and any error that occurs. +func GetDeleteProgress(resource Resource, getError error) (bool, error) { + if deletionTimes, ok := getDeleteTimes(resource); ok { + if !deletionTimes.Verified.IsZero() { + if getError == nil || !apierrors.IsNotFound(getError) { + klog.Warningf("%s has reappeared after having been deleted at %s.", resource, deletionTimes.Verified) + } + } else { + if apierrors.IsNotFound(getError) { + SetDeleteVerified(resource) + } else { + if deletionTimes.Expected != nil { + klog.V(4).Infof("Delete of %s is expected by %s.", resource, deletionTimes.Expected.String()) + } else { + klog.V(4).Infof("Delete of %s has already been requested.", resource) + } + } + } + return true, nil + } + // During an upgrade CVO restarts one or more times. The resource may have been deleted during one of the + // previous CVO life cycles. Simply set the resource as delete verified and log warning. + if apierrors.IsNotFound(getError) { + setDeleteRequestedAndVerified(resource) + return true, nil + } + if getError != nil { + return false, fmt.Errorf("Cannot get %s to delete, err=%v.", resource, getError) + } + return false, nil +} + +// DeletesInProgress returns the set of resources for which deletion has been requested but not yet verified. +func DeletesInProgress() []string { + deletedResources.lock.Lock() + defer deletedResources.lock.Unlock() + deletes := make([]string, 0, len(deletedResources.m)) + for k := range deletedResources.m { + if deletedResources.m[k].Verified.IsZero() { + deletes = append(deletes, k.String()) + } + } + return deletes +} diff --git a/lib/resourcedelete/helper_test.go b/lib/resourcedelete/helper_test.go new file mode 100644 index 000000000..7a5d08be1 --- /dev/null +++ b/lib/resourcedelete/helper_test.go @@ -0,0 +1,119 @@ +package resourcedelete + +import ( + "testing" + "time" +) + +func TestValidDeleteAnnotation(t *testing.T) { + tests := []struct { + name string + annos map[string]string + wantValid bool + wantErr bool + }{ + {name: "no delete annotation", + annos: map[string]string{"foo": "bar"}, + wantValid: false, + wantErr: false}, + {name: "no annotations", + wantValid: false, + wantErr: false}, + {name: "valid delete annotation", + annos: map[string]string{"release.openshift.io/delete": "true"}, + wantValid: true, + wantErr: false}, + {name: "invalid delete annotation", + annos: map[string]string{"release.openshift.io/delete": "false"}, + wantValid: true, + wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + valid, err := ValidDeleteAnnotation(test.annos) + if (err != nil) != test.wantErr { + t.Errorf("ValidDeleteAnnotation{} error = %v, wantErr %v", err, test.wantErr) + } + if valid != test.wantValid { + t.Errorf("ValidDeleteAnnotation{} valid = %v, wantValid %v", valid, test.wantValid) + } + }) + } +} + +func TestSetDeleteVerified(t *testing.T) { + tests := []struct { + name string + resource Resource + wantFound bool + }{ + {name: "set delete verified", + resource: Resource{Kind: "namespace", + Namespace: "", + Name: "openshift-marketplace"}, + + wantFound: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + SetDeleteVerified(test.resource) + times, found := getDeleteTimes(test.resource) + if found != test.wantFound { + t.Errorf("SetDeleteVerified{} found = %v, wantFound %v", found, test.wantFound) + } + if times.Verified.IsZero() { + t.Errorf("SetDeleteVerified{} resource's Verified time is zero") + } + }) + } +} + +func TestDeletesInProgress(t *testing.T) { + tests := []struct { + name string + resources []Resource + delTime deleteTimes + wantInProgress []string + }{ + {name: "2 deletes in progress", + resources: []Resource{ + {Kind: "service", + Namespace: "foo", + Name: "bar"}, + Resource{Kind: "deployment", + Namespace: "openshift-cluster-version", + Name: "cvo"}}, + delTime: deleteTimes{ + Requested: time.Now()}, + wantInProgress: []string{"deployment \"openshift-cluster-version/cvo\"", + "service \"foo/bar\""}}, + {name: "no deletes in progress", + resources: []Resource{ + {Kind: "service", + Namespace: "foo", + Name: "bar"}, + Resource{Kind: "deployment", + Namespace: "openshift-cluster-version", + Name: "cvo"}}, + delTime: deleteTimes{ + Requested: time.Now(), + Verified: time.Now()}}, + } + for _, test := range tests { + for _, r := range test.resources { + deletedResources.m[r] = test.delTime + } + t.Run(test.name, func(t *testing.T) { + deletes := DeletesInProgress() + if len(deletes) != len(test.wantInProgress) { + t.Errorf("DeletesInProgress{} deletes in progress should be %d, found = %d", len(test.wantInProgress), len(deletes)) + } + for _, s := range deletes { + if s != test.wantInProgress[0] && s != test.wantInProgress[1] { + t.Errorf("DeletesInProgress{} delete in progress %s not found. Should be either %s or %s.", + s, test.wantInProgress[0], test.wantInProgress[1]) + } + } + }) + } +} diff --git a/lib/resourcedelete/rbac.go b/lib/resourcedelete/rbac.go new file mode 100644 index 000000000..2beed1d28 --- /dev/null +++ b/lib/resourcedelete/rbac.go @@ -0,0 +1,134 @@ +package resourcedelete + +import ( + "context" + "fmt" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + rbacclientv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" +) + +// DeleteClusterRoleBindingv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteClusterRoleBindingv1(ctx context.Context, client rbacclientv1.ClusterRoleBindingsGetter, required *rbacv1.ClusterRoleBinding, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "clusterrolebinding", + Namespace: "", + Name: required.Name, + } + existing, err := client.ClusterRoleBindings().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.ClusterRoleBindings().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteClusterRolev1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteClusterRolev1(ctx context.Context, client rbacclientv1.ClusterRolesGetter, required *rbacv1.ClusterRole, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "clusterrole", + Namespace: "", + Name: required.Name, + } + existing, err := client.ClusterRoles().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.ClusterRoles().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteRoleBindingv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteRoleBindingv1(ctx context.Context, client rbacclientv1.RoleBindingsGetter, required *rbacv1.RoleBinding, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "rolebinding", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.RoleBindings(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.RoleBindings(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + +// DeleteRolev1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteRolev1(ctx context.Context, client rbacclientv1.RolesGetter, required *rbacv1.Role, + updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "role", + Namespace: required.Namespace, + Name: required.Name, + } + existing, err := client.Roles(required.Namespace).Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Roles(required.Namespace).Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/lib/resourcedelete/security.go b/lib/resourcedelete/security.go new file mode 100644 index 000000000..6366797cc --- /dev/null +++ b/lib/resourcedelete/security.go @@ -0,0 +1,41 @@ +package resourcedelete + +import ( + "context" + "fmt" + + securityv1 "github.com/openshift/api/security/v1" + securityclientv1 "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeleteSecurityContextConstraintsv1 checks the given resource for a valid delete annotation. If found +// and in UpdatingMode it will issue a delete request or provide status of a previousily issued delete request. +// If not in UpdatingMode it simply returns an indication that the delete annotation was found. An error is +// returned if an invalid annotation is found or an error occurs during delete processing. +func DeleteSecurityContextConstraintsv1(ctx context.Context, client securityclientv1.SecurityContextConstraintsGetter, + required *securityv1.SecurityContextConstraints, updateMode bool) (bool, error) { + + if delAnnoFound, err := ValidDeleteAnnotation(required.Annotations); !delAnnoFound || err != nil { + return delAnnoFound, err + } else if !updateMode { + return true, nil + } + resource := Resource{ + Kind: "securitycontextconstraints", + Namespace: "", + Name: required.Name, + } + existing, err := client.SecurityContextConstraints().Get(ctx, required.Name, metav1.GetOptions{}) + if deleteRequested, err := GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.SecurityContextConstraints().Delete(ctx, required.Name, metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} diff --git a/pkg/cvo/internal/generic.go b/pkg/cvo/internal/generic.go index bcf95b6ec..ba4305f2a 100644 --- a/pkg/cvo/internal/generic.go +++ b/pkg/cvo/internal/generic.go @@ -19,6 +19,7 @@ import ( "github.com/openshift/client-go/config/clientset/versioned/scheme" "github.com/openshift/cluster-version-operator/lib/resourcebuilder" + "github.com/openshift/cluster-version-operator/lib/resourcedelete" "github.com/openshift/library-go/pkg/manifest" ) @@ -31,6 +32,32 @@ func readUnstructuredV1OrDie(objBytes []byte) *unstructured.Unstructured { return udi.(*unstructured.Unstructured) } +func deleteUnstructured(ctx context.Context, client dynamic.ResourceInterface, required *unstructured.Unstructured) (bool, error) { + if required.GetName() == "" { + return false, fmt.Errorf("Error running delete, invalid object: name cannot be empty") + } + if delAnnoFound, err := resourcedelete.ValidDeleteAnnotation(required.GetAnnotations()); !delAnnoFound || err != nil { + return delAnnoFound, err + } + resource := resourcedelete.Resource{ + Kind: required.GetKind(), + Namespace: required.GetNamespace(), + Name: required.GetName(), + } + existing, err := client.Get(ctx, required.GetName(), metav1.GetOptions{}) + if deleteRequested, err := resourcedelete.GetDeleteProgress(resource, err); err == nil { + if !deleteRequested { + if err := client.Delete(ctx, required.GetName(), metav1.DeleteOptions{}); err != nil { + return true, fmt.Errorf("Delete request for %s failed, err=%v", resource, err) + } + resourcedelete.SetDeleteRequested(existing, resource) + } + } else { + return true, fmt.Errorf("Error running delete for %s, err=%v", resource, err) + } + return true, nil +} + func applyUnstructured(ctx context.Context, client dynamic.ResourceInterface, required *unstructured.Unstructured) (*unstructured.Unstructured, bool, error) { if required.GetName() == "" { return nil, false, fmt.Errorf("invalid object: name cannot be empty") @@ -117,7 +144,12 @@ func (b *genericBuilder) Do(ctx context.Context) error { b.modifier(ud) } - _, _, err := applyUnstructured(ctx, b.client, ud) + deleteReq, err := deleteUnstructured(ctx, b.client, ud) + if err != nil { + return err + } else if !deleteReq { + _, _, err = applyUnstructured(ctx, b.client, ud) + } return err } diff --git a/pkg/cvo/upgradeable.go b/pkg/cvo/upgradeable.go index 253aed0ba..be92e790a 100644 --- a/pkg/cvo/upgradeable.go +++ b/pkg/cvo/upgradeable.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" + "github.com/openshift/cluster-version-operator/lib/resourcedelete" "github.com/openshift/cluster-version-operator/lib/resourcemerge" ) @@ -215,9 +216,28 @@ func (check *clusterVersionOverridesUpgradeable) Check() *configv1.ClusterOperat return cond } +type clusterManifestDeleteInProgressUpgradeable struct { +} + +func (check *clusterManifestDeleteInProgressUpgradeable) Check() *configv1.ClusterOperatorStatusCondition { + cond := &configv1.ClusterOperatorStatusCondition{ + Type: configv1.ClusterStatusConditionType("UpgradeableDeletesInProgress"), + Status: configv1.ConditionFalse, + } + if deletes := resourcedelete.DeletesInProgress(); len(deletes) > 0 { + resources := strings.Join(deletes, ",") + klog.V(4).Infof("Resource deletions in progress; resources=%s", resources) + cond.Reason = "ResourceDeletesInProgress" + cond.Message = fmt.Sprintf("Cluster minor level upgrades are not allowed while resource deletions are in progress; resources=%s", resources) + return cond + } + return nil +} + func (optr *Operator) defaultUpgradeableChecks() []upgradeableCheck { return []upgradeableCheck{ &clusterOperatorsUpgradeable{coLister: optr.coLister}, &clusterVersionOverridesUpgradeable{name: optr.name, cvLister: optr.cvLister}, + &clusterManifestDeleteInProgressUpgradeable{}, } }