Skip to content

Commit

Permalink
Skip deleting pods/services already in termination
Browse files Browse the repository at this point in the history
Kubernetes pkg/controller/controller_utils.go doesn’t have this logic. This is from kubeflow/training-operator/pull/998. I think it’s safe to keep the logic here.
  • Loading branch information
Jeffwan committed Apr 15, 2020
1 parent 088f125 commit e518b2e
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 0 deletions.
12 changes: 12 additions & 0 deletions pkg/controller.v1/control/pod_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"fmt"
commonutil "github.com/kubeflow/common/pkg/util"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -166,6 +167,17 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime
return fmt.Errorf("object does not have ObjectMeta, %v", err)
}
logger := commonutil.LoggerForJob(accessor)
pod, err := r.KubeClient.CoreV1().Pods(namespace).Get(podID, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return err
}
if pod.DeletionTimestamp != nil {
logger.Infof("pod %s/%s is terminating, skip deleting", pod.Namespace, pod.Name)
return nil
}
logger.Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID)
if err := r.KubeClient.CoreV1().Pods(namespace).Delete(podID, nil); err != nil {
r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err)
Expand Down
12 changes: 12 additions & 0 deletions pkg/controller.v1/control/service_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package control

import (
"fmt"
"k8s.io/apimachinery/pkg/api/errors"
"sync"

log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -112,6 +113,17 @@ func (r RealServiceControl) DeleteService(namespace, serviceID string, object ru
if err != nil {
return fmt.Errorf("object does not have ObjectMeta, %v", err)
}
service, err := r.KubeClient.CoreV1().Services(namespace).Get(serviceID, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return err
}
if service.DeletionTimestamp != nil {
log.Infof("service %s/%s is terminating, skip deleting", service.Namespace, service.Name)
return nil
}
log.Infof("Controller %v deleting service %v/%v", accessor.GetName(), namespace, serviceID)
if err := r.KubeClient.CoreV1().Services(namespace).Delete(serviceID, nil); err != nil {
r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeleteServiceReason, "Error deleting: %v", err)
Expand Down

0 comments on commit e518b2e

Please sign in to comment.