Skip to content

Commit

Permalink
Add a flag to run with --auto-node
Browse files Browse the repository at this point in the history
  • Loading branch information
@enxebre
enxebre committed Dec 17, 2024
1 parent 258e974 commit 3ef7e84
Show file tree
Hide file tree
Showing 4 changed files with 255 additions and 5 deletions.
16 changes: 16 additions & 0 deletions cmd/cluster/aws/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type RawCreateOptions struct {
VPCOwnerCredentials awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool
PublicOnly bool
AutoNode bool
}

// validatedCreateOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked.
Expand Down Expand Up @@ -239,6 +240,19 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e
EndpointAccess: endpointAccess,
},
}
if o.AutoNode {
cluster.Spec.AutoNode = &hyperv1.AutoNode{
Provisioner: &hyperv1.ProvisionerConfig{
Name: hyperv1.ProvisionerKarpeneter,
Karpenter: &hyperv1.KarpenterConfig{
Platform: hyperv1.AWSPlatform,
AWS: &hyperv1.KarpenterAWSConfig{
RoleARN: o.iamInfo.KarpenterRoleARN,
},
},
},
}
}

if o.iamInfo.SharedIngressRoleARN != "" && o.iamInfo.SharedControlPlaneRoleARN != "" {
cluster.Spec.Platform.AWS.SharedVPC = &hyperv1.AWSSharedVPC{
Expand Down Expand Up @@ -389,6 +403,7 @@ func bindCoreOptions(opts *RawCreateOptions, flags *flag.FlagSet) {
flags.StringVar(&opts.CredentialSecretName, "secret-creds", opts.CredentialSecretName, "A Kubernetes secret with needed AWS platform credentials: sts-creds, pull-secret, and a base-domain value. The secret must exist in the supplied \"--namespace\". If a value is provided through the flag '--pull-secret', that value will override the pull-secret value in 'secret-creds'.")
flags.StringVar(&opts.IssuerURL, "oidc-issuer-url", "", "The OIDC provider issuer URL")
flags.BoolVar(&opts.MultiArch, "multi-arch", opts.MultiArch, "If true, this flag indicates the Hosted Cluster will support multi-arch NodePools and will perform additional validation checks to ensure a multi-arch release image or stream was used.")
flags.BoolVar(&opts.AutoNode, "auto-node", opts.AutoNode, "If true, this flag indicates the Hosted Cluster will support AutoNode feature.")
flags.StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC (mask must be 16)")
flags.BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account")
flags.BoolVar(&opts.PublicOnly, "public-only", opts.PublicOnly, "If true, creates a cluster that does not have private subnets or NAT gateway and assigns public IPs to all instances.")
Expand Down Expand Up @@ -467,6 +482,7 @@ func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInf
KMSKeyARN: awsOpts.EtcdKMSKeyARN,
VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials,
PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount,
CreateKarpenterRoleARN: awsOpts.AutoNode,
}
}

Expand Down
5 changes: 4 additions & 1 deletion cmd/infra/aws/create_iam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ type CreateIAMOptions struct {

CredentialsSecretData *util.CredentialsSecretData

additionalIAMTags []*iam.Tag
additionalIAMTags []*iam.Tag
CreateKarpenterRoleARN bool
}

type CreateIAMOutput struct {
Expand All @@ -55,6 +56,8 @@ type CreateIAMOutput struct {

SharedIngressRoleARN string `json:"sharedIngressRoleARN,omitempty"`
SharedControlPlaneRoleARN string `json:"sharedControlPlaneRoleARN,omitempty"`

KarpenterRoleARN string `json:"karpenterRoleARN,omitempty"`
}

func NewCreateIAMCommand() *cobra.Command {
Expand Down
17 changes: 13 additions & 4 deletions cmd/infra/aws/ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import (
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/go-logr/logr"
"github.com/openshift/hypershift/cmd/util"

"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"k8s.io/utils/ptr"
)

const (
Expand Down Expand Up @@ -215,14 +215,20 @@ func (o *CreateInfraOptions) existingDHCPOptions(client ec2iface.EC2API) (string
}

func (o *CreateInfraOptions) CreatePrivateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) {
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB)
karpenterDiscoveryTag := []*ec2.Tag{
{
Key: ptr.To("karpenter.sh/discovery"),
Value: ptr.To(o.InfraID),
},
}
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB, karpenterDiscoveryTag)
}

func (o *CreateInfraOptions) CreatePublicSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) {
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB)
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB, nil)
}

func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string) (string, error) {
func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string, additionalTags []*ec2.Tag) (string, error) {
subnetID, err := o.existingSubnet(client, name)
if err != nil {
return "", err
Expand All @@ -236,6 +242,9 @@ func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API,
Key: aws.String(scopeTag),
Value: aws.String("1"),
})
if additionalTags != nil {
tagSpec[0].Tags = append(tagSpec[0].Tags, additionalTags...)
}

result, err := client.CreateSubnet(&ec2.CreateSubnetInput{
AvailabilityZone: aws.String(zone),
Expand Down
222 changes: 222 additions & 0 deletions cmd/infra/aws/iam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,222 @@ var (
}`,
}

// {
// "Action": [
// "*"
// ],
// "Resource": [
// "*"
// ],
// "Effect": "Allow"
// }
karpenterPolicy = policyBinding{
name: "karpenter",
serviceAccounts: []string{"system:serviceaccount:kube-system:karpenter"},
policy: `{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowScopedEC2InstanceAccessActions",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*::image/*",
"arn:*:ec2:*::snapshot/*",
"arn:*:ec2:*:*:security-group/*",
"arn:*:ec2:*:*:subnet/*"
],
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet"
]
},
{
"Sid": "AllowScopedEC2LaunchTemplateAccessActions",
"Effect": "Allow",
"Resource": "arn:*:ec2:*:*:launch-template/*",
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet"
]
},
{
"Sid": "AllowScopedEC2InstanceActionsWithTags",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:fleet/*",
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:volume/*",
"arn:*:ec2:*:*:network-interface/*",
"arn:*:ec2:*:*:launch-template/*",
"arn:*:ec2:*:*:spot-instances-request/*"
],
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet",
"ec2:CreateLaunchTemplate"
],
"Condition": {
"StringLike": {
"aws:RequestTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedResourceCreationTagging",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:fleet/*",
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:volume/*",
"arn:*:ec2:*:*:network-interface/*",
"arn:*:ec2:*:*:launch-template/*",
"arn:*:ec2:*:*:spot-instances-request/*"
],
"Action": "ec2:CreateTags",
"Condition": {
"StringEquals": {
"ec2:CreateAction": [
"RunInstances",
"CreateFleet",
"CreateLaunchTemplate"
]
},
"StringLike": {
"aws:RequestTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedResourceTagging",
"Effect": "Allow",
"Resource": "arn:*:ec2:*:*:instance/*",
"Action": "ec2:CreateTags",
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedDeletion",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:launch-template/*"
],
"Action": [
"ec2:TerminateInstances",
"ec2:DeleteLaunchTemplate"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowRegionalReadActions",
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:DescribeImages",
"ec2:DescribeInstances",
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeInstanceTypes",
"ec2:DescribeLaunchTemplates",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets"
]
},
{
"Sid": "AllowSSMReadActions",
"Effect": "Allow",
"Resource": "arn:*:ssm:*::parameter/aws/service/*",
"Action": "ssm:GetParameter"
},
{
"Sid": "AllowPricingReadActions",
"Effect": "Allow",
"Resource": "*",
"Action": "pricing:GetProducts"
},
{
"Sid": "AllowInterruptionQueueActions",
"Effect": "Allow",
"Resource": "*",
"Action": [
"sqs:DeleteMessage",
"sqs:GetQueueUrl",
"sqs:ReceiveMessage"
]
},
{
"Sid": "AllowPassingInstanceRole",
"Effect": "Allow",
"Resource": "arn:*:iam::*:role/*",
"Action": "iam:PassRole",
"Condition": {
"StringEquals": {
"iam:PassedToService": [
"ec2.amazonaws.com",
"ec2.amazonaws.com.cn"
]
}
}
},
{
"Sid": "AllowScopedInstanceProfileCreationActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:CreateInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowScopedInstanceProfileTagActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:TagInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*",
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowScopedInstanceProfileActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:AddRoleToInstanceProfile",
"iam:RemoveRoleFromInstanceProfile",
"iam:DeleteInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowInstanceProfileReadActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": "iam:GetInstanceProfile"
}
]
}`,
}

nodePoolPolicy = policyBinding{
name: "node-pool",
serviceAccounts: []string{"system:serviceaccount:kube-system:capa-controller-manager"},
Expand Down Expand Up @@ -628,6 +844,12 @@ func (o *CreateIAMOptions) CreateOIDCResources(iamClient iamiface.IAMAPI, logger
&output.Roles.ControlPlaneOperatorARN: controlPlaneOperatorPolicy(o.LocalZoneID, sharedVPC),
&output.Roles.NetworkARN: cloudNetworkConfigControllerPolicy,
}

if o.CreateKarpenterRoleARN {
bindings[&output.KarpenterRoleARN] = karpenterPolicy

}

if len(o.KMSKeyARN) > 0 {
bindings[&output.KMSProviderRoleARN] = kmsProviderPolicy(o.KMSKeyARN)
}
Expand Down

0 comments on commit 3ef7e84

Please sign in to comment.