Skip to content

Commit

Permalink
Add CLI flag to support shared vpc private zones in cluster account
Browse files Browse the repository at this point in the history 
Adds a flag `--private-zones-in-cluster-account` to the following commands:
hypershift create infra aws
hypershift create iam aws
hypershift create cluster aws
hypershift destroy infra aws
hypershift destroy iam aws
hypershift destroy cluster aws

The flag enables private hosted zones to be created in the cluster
creator account in a shared vpc infrastructure.
  • Loading branch information
@csrwng
csrwng committed Nov 14, 2024
1 parent 5f4541a commit dcdf858
Show file tree
Hide file tree
Showing 10 changed files with 259 additions and 153 deletions.
96 changes: 50 additions & 46 deletions cmd/cluster/aws/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,28 +23,29 @@ import (
)

type RawCreateOptions struct {
Credentials awsutil.AWSCredentialsOptions
CredentialSecretName string
AdditionalTags []string
IAMJSON string
InstanceType string
IssuerURL string
PrivateZoneID string
PublicZoneID string
Region string
RootVolumeIOPS int64
RootVolumeSize int64
RootVolumeType string
RootVolumeEncryptionKey string
EndpointAccess string
Zones []string
EtcdKMSKeyARN string
EnableProxy bool
ProxyVPCEndpointServiceName string
SingleNATGateway bool
MultiArch bool
VPCCIDR string
VPCOwnerCredentials awsutil.AWSCredentialsOptions
Credentials awsutil.AWSCredentialsOptions
CredentialSecretName string
AdditionalTags []string
IAMJSON string
InstanceType string
IssuerURL string
PrivateZoneID string
PublicZoneID string
Region string
RootVolumeIOPS int64
RootVolumeSize int64
RootVolumeType string
RootVolumeEncryptionKey string
EndpointAccess string
Zones []string
EtcdKMSKeyARN string
EnableProxy bool
ProxyVPCEndpointServiceName string
SingleNATGateway bool
MultiArch bool
VPCCIDR string
VPCOwnerCredentials awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool
}

// validatedCreateOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked.
Expand Down Expand Up @@ -381,6 +382,7 @@ func bindCoreOptions(opts *RawCreateOptions, flags *flag.FlagSet) {
flags.StringVar(&opts.IssuerURL, "oidc-issuer-url", "", "The OIDC provider issuer URL")
flags.BoolVar(&opts.MultiArch, "multi-arch", opts.MultiArch, "If true, this flag indicates the Hosted Cluster will support multi-arch NodePools and will perform additional validation checks to ensure a multi-arch release image or stream was used.")
flags.StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC (mask must be 16)")
flags.BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account")

_ = flags.MarkDeprecated("multi-arch", "Multi-arch validation is now performed automatically based on the release image and signaled in the HostedCluster.Status.PayloadArch.")
}
Expand Down Expand Up @@ -424,35 +426,37 @@ func NewCreateCommand(opts *core.RawCreateOptions) *cobra.Command {

func CreateInfraOptions(awsOpts *ValidatedCreateOptions, opts *core.CreateOptions) awsinfra.CreateInfraOptions {
return awsinfra.CreateInfraOptions{
Region: awsOpts.Region,
InfraID: opts.InfraID,
AWSCredentialsOpts: awsOpts.Credentials,
Name: opts.Name,
BaseDomain: opts.BaseDomain,
BaseDomainPrefix: opts.BaseDomainPrefix,
AdditionalTags: awsOpts.AdditionalTags,
Zones: awsOpts.Zones,
EnableProxy: awsOpts.EnableProxy,
ProxyVPCEndpointServiceName: awsOpts.ProxyVPCEndpointServiceName,
SSHKeyFile: opts.SSHKeyFile,
SingleNATGateway: awsOpts.SingleNATGateway,
VPCCIDR: awsOpts.VPCCIDR,
VPCOwnerCredentialOpts: awsOpts.VPCOwnerCredentials,
Region: awsOpts.Region,
InfraID: opts.InfraID,
AWSCredentialsOpts: awsOpts.Credentials,
Name: opts.Name,
BaseDomain: opts.BaseDomain,
BaseDomainPrefix: opts.BaseDomainPrefix,
AdditionalTags: awsOpts.AdditionalTags,
Zones: awsOpts.Zones,
EnableProxy: awsOpts.EnableProxy,
ProxyVPCEndpointServiceName: awsOpts.ProxyVPCEndpointServiceName,
SSHKeyFile: opts.SSHKeyFile,
SingleNATGateway: awsOpts.SingleNATGateway,
VPCCIDR: awsOpts.VPCCIDR,
VPCOwnerCredentialOpts: awsOpts.VPCOwnerCredentials,
PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount,
}
}

func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInfraOutput) awsinfra.CreateIAMOptions {
return awsinfra.CreateIAMOptions{
Region: awsOpts.Region,
AWSCredentialsOpts: awsOpts.Credentials,
InfraID: infra.InfraID,
IssuerURL: awsOpts.IssuerURL,
AdditionalTags: awsOpts.AdditionalTags,
PrivateZoneID: infra.PrivateZoneID,
PublicZoneID: infra.PublicZoneID,
LocalZoneID: infra.LocalZoneID,
KMSKeyARN: awsOpts.EtcdKMSKeyARN,
VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials,
Region: awsOpts.Region,
AWSCredentialsOpts: awsOpts.Credentials,
InfraID: infra.InfraID,
IssuerURL: awsOpts.IssuerURL,
AdditionalTags: awsOpts.AdditionalTags,
PrivateZoneID: infra.PrivateZoneID,
PublicZoneID: infra.PublicZoneID,
LocalZoneID: infra.LocalZoneID,
KMSKeyARN: awsOpts.EtcdKMSKeyARN,
VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials,
PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount,
}
}

Expand Down
35 changes: 19 additions & 16 deletions cmd/cluster/aws/destroy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ func NewDestroyCommand(opts *core.DestroyOptions) *cobra.Command {
cmd.Flags().StringVar(&opts.AWSPlatform.BaseDomainPrefix, "base-domain-prefix", opts.AWSPlatform.BaseDomainPrefix, "Cluster's base domain prefix; inferred from the hosted cluster by default")
cmd.Flags().StringVar(&opts.CredentialSecretName, "secret-creds", opts.CredentialSecretName, "A Kubernetes secret with a platform credential, pull-secret and base-domain. The secret must exist in the supplied \"--namespace\"")
cmd.Flags().DurationVar(&opts.AWSPlatform.AwsInfraGracePeriod, "aws-infra-grace-period", opts.AWSPlatform.AwsInfraGracePeriod, "Timeout for destroying infrastructure in minutes")
cmd.Flags().BoolVar(&opts.AWSPlatform.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.AWSPlatform.PrivateZonesInClusterAccount, "In shared VPC infrastructure, delete private hosted zones in cluster account")

opts.AWSPlatform.Credentials.BindFlags(cmd.Flags())
opts.AWSPlatform.VPCOwnerCredentials.BindVPCOwnerFlags(cmd.Flags())
Expand Down Expand Up @@ -75,16 +76,17 @@ func destroyPlatformSpecifics(ctx context.Context, o *core.DestroyOptions) error

o.Log.Info("Destroying infrastructure", "infraID", infraID)
destroyInfraOpts := awsinfra.DestroyInfraOptions{
Region: region,
InfraID: infraID,
AWSCredentialsOpts: &awsinfra.DelegatedAWSCredentialOptions{AWSCredentialsOpts: &o.AWSPlatform.Credentials},
Name: o.Name,
BaseDomain: baseDomain,
BaseDomainPrefix: baseDomainPrefix,
AwsInfraGracePeriod: o.AWSPlatform.AwsInfraGracePeriod,
Log: o.Log,
CredentialsSecretData: secretData,
VPCOwnerCredentialsOpts: o.AWSPlatform.VPCOwnerCredentials,
Region: region,
InfraID: infraID,
AWSCredentialsOpts: &awsinfra.DelegatedAWSCredentialOptions{AWSCredentialsOpts: &o.AWSPlatform.Credentials},
Name: o.Name,
BaseDomain: baseDomain,
BaseDomainPrefix: baseDomainPrefix,
AwsInfraGracePeriod: o.AWSPlatform.AwsInfraGracePeriod,
Log: o.Log,
CredentialsSecretData: secretData,
VPCOwnerCredentialsOpts: o.AWSPlatform.VPCOwnerCredentials,
PrivateZonesInClusterAccount: o.AWSPlatform.PrivateZonesInClusterAccount,
}
if err := destroyInfraOpts.Run(ctx); err != nil {
return fmt.Errorf("failed to destroy infrastructure: %w", err)
Expand All @@ -93,12 +95,13 @@ func destroyPlatformSpecifics(ctx context.Context, o *core.DestroyOptions) error
if !o.AWSPlatform.PreserveIAM {
o.Log.Info("Destroying IAM", "infraID", infraID)
destroyOpts := awsinfra.DestroyIAMOptions{
Region: region,
AWSCredentialsOpts: o.AWSPlatform.Credentials,
InfraID: infraID,
Log: o.Log,
CredentialsSecretData: secretData,
VPCOwnerCredentialsOpts: o.AWSPlatform.VPCOwnerCredentials,
Region: region,
AWSCredentialsOpts: o.AWSPlatform.Credentials,
InfraID: infraID,
Log: o.Log,
CredentialsSecretData: secretData,
VPCOwnerCredentialsOpts: o.AWSPlatform.VPCOwnerCredentials,
PrivateZonesInClusterAccount: o.AWSPlatform.PrivateZonesInClusterAccount,
}
if err := destroyOpts.Run(ctx); err != nil {
return fmt.Errorf("failed to destroy IAM: %w", err)
Expand Down
17 changes: 9 additions & 8 deletions cmd/cluster/core/destroy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,15 @@ type DestroyOptions struct {
}

type AWSPlatformDestroyOptions struct {
Credentials awsutil.AWSCredentialsOptions
BaseDomain string
BaseDomainPrefix string
PreserveIAM bool
Region string
PostDeleteAction func()
AwsInfraGracePeriod time.Duration
VPCOwnerCredentials awsutil.AWSCredentialsOptions
Credentials awsutil.AWSCredentialsOptions
BaseDomain string
BaseDomainPrefix string
PreserveIAM bool
Region string
PostDeleteAction func()
AwsInfraGracePeriod time.Duration
VPCOwnerCredentials awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool
}

type AzurePlatformDestroyOptions struct {
Expand Down
35 changes: 28 additions & 7 deletions cmd/infra/aws/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ type CreateInfraOptions struct {

CredentialsSecretData *util.CredentialsSecretData

VPCOwnerCredentialOpts awsutil.AWSCredentialsOptions
VPCOwnerCredentialOpts awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool

additionalEC2Tags []*ec2.Tag
}
Expand Down Expand Up @@ -109,6 +110,7 @@ func NewCreateCommand() *cobra.Command {
cmd.Flags().StringVar(&opts.ProxyVPCEndpointServiceName, "proxy-vpc-endpoint-service-name", opts.ProxyVPCEndpointServiceName, "The name of a VPC Endpoint Service offering a proxy service to use for the cluster")
cmd.Flags().BoolVar(&opts.SingleNATGateway, "single-nat-gateway", opts.SingleNATGateway, "If enabled, only a single NAT gateway is created, even if multiple zones are specified")
cmd.Flags().StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC")
cmd.Flags().BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account")

cmd.MarkFlagRequired("infra-id")
cmd.MarkFlagRequired("base-domain")
Expand Down Expand Up @@ -186,11 +188,13 @@ func (o *CreateInfraOptions) CreateInfra(ctx context.Context, l logr.Logger) (*C
}
}

var ec2Client *ec2.EC2
var clusterCreatorEC2Client, ec2Client *ec2.EC2
var vpcOwnerRoute53Client, route53Client *route53.Route53
ec2Client = ec2.New(awsSession, awsutil.NewConfig())
clusterCreatorEC2Client = ec2.New(awsSession, awsutil.NewConfig())
if vpcOwnerAWSSession != nil {
ec2Client = ec2.New(vpcOwnerAWSSession, awsutil.NewConfig())
} else {
ec2Client = clusterCreatorEC2Client
}
route53Client = route53.New(awsSession, awsutil.NewAWSRoute53Config())
if vpcOwnerAWSSession != nil {
Expand Down Expand Up @@ -290,17 +294,34 @@ func (o *CreateInfraOptions) CreateInfra(ctx context.Context, l logr.Logger) (*C
return nil, err
}

result.PrivateZoneID, err = o.CreatePrivateZone(ctx, l, vpcOwnerRoute53Client, ZoneName(o.Name, o.BaseDomainPrefix, o.BaseDomain), result.VPCID)
if vpcOwnerAWSSession != nil {
if err := o.shareSubnets(ctx, l, vpcOwnerAWSSession, awsSession, publicSubnetIDs, result); err != nil {
return nil, err
}
}

privateZoneClient := vpcOwnerRoute53Client
var initialVPC string
if o.PrivateZonesInClusterAccount {
privateZoneClient = route53Client

// Create a dummy vpc that we can use to create the private hosted zones
if initialVPC, err = o.createVPC(l, clusterCreatorEC2Client); err != nil {
return nil, err
}
}

result.PrivateZoneID, err = o.CreatePrivateZone(ctx, l, privateZoneClient, ZoneName(o.Name, o.BaseDomainPrefix, o.BaseDomain), result.VPCID, o.PrivateZonesInClusterAccount, vpcOwnerRoute53Client, initialVPC)
if err != nil {
return nil, err
}
result.LocalZoneID, err = o.CreatePrivateZone(ctx, l, vpcOwnerRoute53Client, fmt.Sprintf("%s.%s", o.Name, hypershiftLocalZoneName), result.VPCID)
result.LocalZoneID, err = o.CreatePrivateZone(ctx, l, privateZoneClient, fmt.Sprintf("%s.%s", o.Name, hypershiftLocalZoneName), result.VPCID, o.PrivateZonesInClusterAccount, vpcOwnerRoute53Client, initialVPC)
if err != nil {
return nil, err
}

if vpcOwnerAWSSession != nil {
if err := o.shareSubnets(ctx, l, vpcOwnerAWSSession, awsSession, publicSubnetIDs, result); err != nil {
if initialVPC != "" {
if err := o.deleteVPC(l, clusterCreatorEC2Client, initialVPC); err != nil {
return nil, err
}
}
Expand Down
12 changes: 11 additions & 1 deletion cmd/infra/aws/create_iam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type CreateIAMOptions struct {
KMSKeyARN string
AdditionalTags []string
VPCOwnerCredentialsOpts awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool

CredentialsSecretData *util.CredentialsSecretData

Expand Down Expand Up @@ -79,6 +80,7 @@ func NewCreateIAMCommand() *cobra.Command {
cmd.Flags().StringVar(&opts.LocalZoneID, "local-zone-id", opts.LocalZoneID, "The id of the clusters local route53 zone")
cmd.Flags().StringVar(&opts.KMSKeyARN, "kms-key-arn", opts.KMSKeyARN, "The ARN of the KMS key to use for Etcd encryption. If not supplied, etcd encryption will default to using a generated AESCBC key.")
cmd.Flags().StringSliceVar(&opts.AdditionalTags, "additional-tags", opts.AdditionalTags, "Additional tags to set on AWS resources")
cmd.Flags().BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account")

opts.AWSCredentialsOpts.BindFlags(cmd.Flags())
opts.VPCOwnerCredentialsOpts.BindVPCOwnerFlags(cmd.Flags())
Expand Down Expand Up @@ -193,7 +195,15 @@ func (o *CreateIAMOptions) CreateIAM(ctx context.Context, client crclient.Client
return nil, err
}
vpcOwnerIAMClient := iam.New(vpcOwnerAWSSession, awsConfig)
if err := o.CreateSharedVPCRoles(vpcOwnerIAMClient, logger, results); err != nil {

route53RoleClient := vpcOwnerIAMClient
if o.PrivateZonesInClusterAccount {
route53RoleClient = iamClient
}
if results.SharedIngressRoleARN, err = o.CreateSharedVPCRoute53Role(route53RoleClient, logger, results.Roles.IngressARN, results.Roles.ControlPlaneOperatorARN); err != nil {
return nil, err
}
if results.SharedControlPlaneRoleARN, err = o.CreateSharedVPCEndpointRole(vpcOwnerIAMClient, logger, results.Roles.ControlPlaneOperatorARN); err != nil {
return nil, err
}
}
Expand Down
Loading

0 comments on commit dcdf858

Please sign in to comment.