diff --git a/cmd/cluster/destroy.go b/cmd/cluster/destroy.go index c45b8dd666..420aec7af3 100644 --- a/cmd/cluster/destroy.go +++ b/cmd/cluster/destroy.go @@ -120,7 +120,9 @@ func DestroyCluster(ctx context.Context, o *DestroyOptions) error { Name: hostedCluster.GetName(), BaseDomain: hostedCluster.Spec.DNS.BaseDomain, } - destroyInfraOpts.Run(ctx) + if err := destroyInfraOpts.Run(ctx); err != nil { + return fmt.Errorf("failed to destroy infrastructure: %w", err) + } if !o.PreserveIAM { log.Info("destroying IAM") @@ -129,8 +131,7 @@ func DestroyCluster(ctx context.Context, o *DestroyOptions) error { AWSCredentialsFile: o.AWSCredentialsFile, InfraID: hostedCluster.Spec.InfraID, } - err := destroyOpts.DestroyIAM(ctx) - if err != nil { + if err := destroyOpts.Run(ctx); err != nil { return fmt.Errorf("failed to destroy IAM: %w", err) } } diff --git a/cmd/infra/aws/destroy.go b/cmd/infra/aws/destroy.go index 92c26b1d66..c6c4a37e04 100644 --- a/cmd/infra/aws/destroy.go +++ b/cmd/infra/aws/destroy.go @@ -3,7 +3,10 @@ package aws import ( "context" "fmt" + "os" + "os/signal" "strings" + "syscall" "time" "github.com/aws/aws-sdk-go/aws" @@ -46,19 +49,29 @@ func NewDestroyCommand() *cobra.Command { cmd.MarkFlagRequired("aws-creds") cmd.MarkFlagRequired("base-domain") - cmd.Run = func(cmd *cobra.Command, args []string) { - opts.Run(context.Background()) + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithCancel(context.Background()) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT) + go func() { + <-sigs + cancel() + }() + if err := opts.Run(ctx); err != nil { + return err + } log.Info("Successfully destroyed AWS infra") + return nil } return cmd } -func (o *DestroyInfraOptions) Run(ctx context.Context) { - wait.PollUntil(5*time.Second, func() (bool, error) { +func (o *DestroyInfraOptions) Run(ctx context.Context) error { + return wait.PollUntil(5*time.Second, func() (bool, error) { err := o.DestroyInfra(ctx) if err != nil { - log.Error(err, "error destroying infra") + log.Info("WARNING: error during destroy, will retry", "error", err) return false, nil } return true, nil diff --git a/cmd/infra/aws/destroy_iam.go b/cmd/infra/aws/destroy_iam.go index 273cad30d4..798d1749ed 100644 --- a/cmd/infra/aws/destroy_iam.go +++ b/cmd/infra/aws/destroy_iam.go @@ -7,6 +7,7 @@ import ( "os/signal" "strings" "syscall" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -15,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" ) type DestroyIAMOptions struct { @@ -50,12 +52,27 @@ func NewDestroyIAMCommand() *cobra.Command { <-sigs cancel() }() - return opts.DestroyIAM(ctx) + if err := opts.DestroyIAM(ctx); err != nil { + return err + } + log.Info("Successfully destroyed IAM infra") + return nil } return cmd } +func (o *DestroyIAMOptions) Run(ctx context.Context) error { + return wait.PollUntil(5*time.Second, func() (bool, error) { + err := o.DestroyIAM(ctx) + if err != nil { + log.Info("WARNING: error during destroy, will retry", "error", err) + return false, nil + } + return true, nil + }, ctx.Done()) +} + func (o *DestroyIAMOptions) DestroyIAM(ctx context.Context) error { var err error iamClient, err := IAMClient(o.AWSCredentialsFile, o.Region)