From 3ef7e84ffd482128102d9a65a7cd49ffb9132b6b Mon Sep 17 00:00:00 2001 From: enxebre Date: Thu, 12 Dec 2024 13:04:03 +0100 Subject: [PATCH] Add a flag to run with --auto-node --- cmd/cluster/aws/create.go | 16 +++ cmd/infra/aws/create_iam.go | 5 +- cmd/infra/aws/ec2.go | 17 ++- cmd/infra/aws/iam.go | 222 ++++++++++++++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 5 deletions(-) diff --git a/cmd/cluster/aws/create.go b/cmd/cluster/aws/create.go index 877ee72f6a..1b0386f915 100644 --- a/cmd/cluster/aws/create.go +++ b/cmd/cluster/aws/create.go @@ -47,6 +47,7 @@ type RawCreateOptions struct { VPCOwnerCredentials awsutil.AWSCredentialsOptions PrivateZonesInClusterAccount bool PublicOnly bool + AutoNode bool } // validatedCreateOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked. @@ -239,6 +240,19 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e EndpointAccess: endpointAccess, }, } + if o.AutoNode { + cluster.Spec.AutoNode = &hyperv1.AutoNode{ + Provisioner: &hyperv1.ProvisionerConfig{ + Name: hyperv1.ProvisionerKarpeneter, + Karpenter: &hyperv1.KarpenterConfig{ + Platform: hyperv1.AWSPlatform, + AWS: &hyperv1.KarpenterAWSConfig{ + RoleARN: o.iamInfo.KarpenterRoleARN, + }, + }, + }, + } + } if o.iamInfo.SharedIngressRoleARN != "" && o.iamInfo.SharedControlPlaneRoleARN != "" { cluster.Spec.Platform.AWS.SharedVPC = &hyperv1.AWSSharedVPC{ @@ -389,6 +403,7 @@ func bindCoreOptions(opts *RawCreateOptions, flags *flag.FlagSet) { flags.StringVar(&opts.CredentialSecretName, "secret-creds", opts.CredentialSecretName, "A Kubernetes secret with needed AWS platform credentials: sts-creds, pull-secret, and a base-domain value. The secret must exist in the supplied \"--namespace\". If a value is provided through the flag '--pull-secret', that value will override the pull-secret value in 'secret-creds'.") flags.StringVar(&opts.IssuerURL, "oidc-issuer-url", "", "The OIDC provider issuer URL") flags.BoolVar(&opts.MultiArch, "multi-arch", opts.MultiArch, "If true, this flag indicates the Hosted Cluster will support multi-arch NodePools and will perform additional validation checks to ensure a multi-arch release image or stream was used.") + flags.BoolVar(&opts.AutoNode, "auto-node", opts.AutoNode, "If true, this flag indicates the Hosted Cluster will support AutoNode feature.") flags.StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC (mask must be 16)") flags.BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account") flags.BoolVar(&opts.PublicOnly, "public-only", opts.PublicOnly, "If true, creates a cluster that does not have private subnets or NAT gateway and assigns public IPs to all instances.") @@ -467,6 +482,7 @@ func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInf KMSKeyARN: awsOpts.EtcdKMSKeyARN, VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials, PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount, + CreateKarpenterRoleARN: awsOpts.AutoNode, } } diff --git a/cmd/infra/aws/create_iam.go b/cmd/infra/aws/create_iam.go index 590c5f1ecd..9d54bd6680 100644 --- a/cmd/infra/aws/create_iam.go +++ b/cmd/infra/aws/create_iam.go @@ -41,7 +41,8 @@ type CreateIAMOptions struct { CredentialsSecretData *util.CredentialsSecretData - additionalIAMTags []*iam.Tag + additionalIAMTags []*iam.Tag + CreateKarpenterRoleARN bool } type CreateIAMOutput struct { @@ -55,6 +56,8 @@ type CreateIAMOutput struct { SharedIngressRoleARN string `json:"sharedIngressRoleARN,omitempty"` SharedControlPlaneRoleARN string `json:"sharedControlPlaneRoleARN,omitempty"` + + KarpenterRoleARN string `json:"karpenterRoleARN,omitempty"` } func NewCreateIAMCommand() *cobra.Command { diff --git a/cmd/infra/aws/ec2.go b/cmd/infra/aws/ec2.go index 24e2d598fa..b3315290f0 100644 --- a/cmd/infra/aws/ec2.go +++ b/cmd/infra/aws/ec2.go @@ -11,9 +11,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/go-logr/logr" "github.com/openshift/hypershift/cmd/util" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" ) const ( @@ -215,14 +215,20 @@ func (o *CreateInfraOptions) existingDHCPOptions(client ec2iface.EC2API) (string } func (o *CreateInfraOptions) CreatePrivateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) { - return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB) + karpenterDiscoveryTag := []*ec2.Tag{ + { + Key: ptr.To("karpenter.sh/discovery"), + Value: ptr.To(o.InfraID), + }, + } + return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB, karpenterDiscoveryTag) } func (o *CreateInfraOptions) CreatePublicSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) { - return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB) + return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB, nil) } -func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string) (string, error) { +func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string, additionalTags []*ec2.Tag) (string, error) { subnetID, err := o.existingSubnet(client, name) if err != nil { return "", err @@ -236,6 +242,9 @@ func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, Key: aws.String(scopeTag), Value: aws.String("1"), }) + if additionalTags != nil { + tagSpec[0].Tags = append(tagSpec[0].Tags, additionalTags...) + } result, err := client.CreateSubnet(&ec2.CreateSubnetInput{ AvailabilityZone: aws.String(zone), diff --git a/cmd/infra/aws/iam.go b/cmd/infra/aws/iam.go index 6de3f23ee5..da410da7d0 100644 --- a/cmd/infra/aws/iam.go +++ b/cmd/infra/aws/iam.go @@ -209,6 +209,222 @@ var ( }`, } + // { + // "Action": [ + // "*" + // ], + // "Resource": [ + // "*" + // ], + // "Effect": "Allow" + // } + karpenterPolicy = policyBinding{ + name: "karpenter", + serviceAccounts: []string{"system:serviceaccount:kube-system:karpenter"}, + policy: `{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowScopedEC2InstanceAccessActions", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*::image/*", + "arn:*:ec2:*::snapshot/*", + "arn:*:ec2:*:*:security-group/*", + "arn:*:ec2:*:*:subnet/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:*:ec2:*:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:fleet/*", + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:volume/*", + "arn:*:ec2:*:*:network-interface/*", + "arn:*:ec2:*:*:launch-template/*", + "arn:*:ec2:*:*:spot-instances-request/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:fleet/*", + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:volume/*", + "arn:*:ec2:*:*:network-interface/*", + "arn:*:ec2:*:*:launch-template/*", + "arn:*:ec2:*:*:spot-instances-request/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:*:ec2:*:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:*:ec2:*:*:instance/*", + "arn:*:ec2:*:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + }, + { + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:*:ssm:*::parameter/aws/service/*", + "Action": "ssm:GetParameter" + }, + { + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" + }, + { + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + }, + { + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:*:iam::*:role/*", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": [ + "ec2.amazonaws.com", + "ec2.amazonaws.com.cn" + ] + } + } + }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "arn:*:iam::*:instance-profile/*", + "Action": "iam:GetInstanceProfile" + } + ] + }`, + } + nodePoolPolicy = policyBinding{ name: "node-pool", serviceAccounts: []string{"system:serviceaccount:kube-system:capa-controller-manager"}, @@ -628,6 +844,12 @@ func (o *CreateIAMOptions) CreateOIDCResources(iamClient iamiface.IAMAPI, logger &output.Roles.ControlPlaneOperatorARN: controlPlaneOperatorPolicy(o.LocalZoneID, sharedVPC), &output.Roles.NetworkARN: cloudNetworkConfigControllerPolicy, } + + if o.CreateKarpenterRoleARN { + bindings[&output.KarpenterRoleARN] = karpenterPolicy + + } + if len(o.KMSKeyARN) > 0 { bindings[&output.KMSProviderRoleARN] = kmsProviderPolicy(o.KMSKeyARN) }