Merge pull request openshift#177 from ironcladlou/destroy-retries

cli: Improve destroy retry handling
gnufied · Apr 22, 2021 · 5243816 · 5243816
2 parents de53a22 + 1daa001
commit 5243816
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 9 deletions.
diff --git a/cmd/cluster/destroy.go b/cmd/cluster/destroy.go
@@ -120,7 +120,9 @@ func DestroyCluster(ctx context.Context, o *DestroyOptions) error {
 		Name:               hostedCluster.GetName(),
 		BaseDomain:         hostedCluster.Spec.DNS.BaseDomain,
 	}
-	destroyInfraOpts.Run(ctx)
+	if err := destroyInfraOpts.Run(ctx); err != nil {
+		return fmt.Errorf("failed to destroy infrastructure: %w", err)
+	}
 
 	if !o.PreserveIAM {
 		log.Info("destroying IAM")
@@ -129,8 +131,7 @@ func DestroyCluster(ctx context.Context, o *DestroyOptions) error {
 			AWSCredentialsFile: o.AWSCredentialsFile,
 			InfraID:            hostedCluster.Spec.InfraID,
 		}
-		err := destroyOpts.DestroyIAM(ctx)
-		if err != nil {
+		if err := destroyOpts.Run(ctx); err != nil {
 			return fmt.Errorf("failed to destroy IAM: %w", err)
 		}
 	}

diff --git a/cmd/infra/aws/destroy.go b/cmd/infra/aws/destroy.go
@@ -3,7 +3,10 @@ package aws
 import (
 	"context"
 	"fmt"
+	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -46,19 +49,29 @@ func NewDestroyCommand() *cobra.Command {
 	cmd.MarkFlagRequired("aws-creds")
 	cmd.MarkFlagRequired("base-domain")
 
-	cmd.Run = func(cmd *cobra.Command, args []string) {
-		opts.Run(context.Background())
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGINT)
+		go func() {
+			<-sigs
+			cancel()
+		}()
+		if err := opts.Run(ctx); err != nil {
+			return err
+		}
 		log.Info("Successfully destroyed AWS infra")
+		return nil
 	}
 
 	return cmd
 }
 
-func (o *DestroyInfraOptions) Run(ctx context.Context) {
-	wait.PollUntil(5*time.Second, func() (bool, error) {
+func (o *DestroyInfraOptions) Run(ctx context.Context) error {
+	return wait.PollUntil(5*time.Second, func() (bool, error) {
 		err := o.DestroyInfra(ctx)
 		if err != nil {
-			log.Error(err, "error destroying infra")
+			log.Info("WARNING: error during destroy, will retry", "error", err)
 			return false, nil
 		}
 		return true, nil

diff --git a/cmd/infra/aws/destroy_iam.go b/cmd/infra/aws/destroy_iam.go
@@ -7,6 +7,7 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -15,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 type DestroyIAMOptions struct {
@@ -50,12 +52,27 @@ func NewDestroyIAMCommand() *cobra.Command {
 			<-sigs
 			cancel()
 		}()
-		return opts.DestroyIAM(ctx)
+		if err := opts.DestroyIAM(ctx); err != nil {
+			return err
+		}
+		log.Info("Successfully destroyed IAM infra")
+		return nil
 	}
 
 	return cmd
 }
 
+func (o *DestroyIAMOptions) Run(ctx context.Context) error {
+	return wait.PollUntil(5*time.Second, func() (bool, error) {
+		err := o.DestroyIAM(ctx)
+		if err != nil {
+			log.Info("WARNING: error during destroy, will retry", "error", err)
+			return false, nil
+		}
+		return true, nil
+	}, ctx.Done())
+}
+
 func (o *DestroyIAMOptions) DestroyIAM(ctx context.Context) error {
 	var err error
 	iamClient, err := IAMClient(o.AWSCredentialsFile, o.Region)