Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HOSTEDCP-2220: Autonode karpenter #5279

Merged
merged 6 commits into from
Dec 17, 2024
Merged

HOSTEDCP-2220: Autonode karpenter #5279

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
18ffc1b
Run make update
enxebre Dec 12, 2024
e949256
Add karpenter operator
enxebre Dec 12, 2024
2c4aee6
Let the HO manage the Karpenter Operator Deployment
enxebre Dec 12, 2024
258e974
Add AutoNode Karpenter feature gated API support
enxebre Dec 12, 2024
3ef7e84
Add a flag to run with --auto-node
enxebre Dec 12, 2024
f596b35
Include karpenter-operator binary within HO image
enxebre Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add a flag to run with --auto-node
  • Loading branch information
@enxebre
enxebre committed Dec 17, 2024
commit 3ef7e84ffd482128102d9a65a7cd49ffb9132b6b
16 changes: 16 additions & 0 deletions cmd/cluster/aws/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type RawCreateOptions struct {
VPCOwnerCredentials awsutil.AWSCredentialsOptions
PrivateZonesInClusterAccount bool
PublicOnly bool
AutoNode bool
}

// validatedCreateOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked.
Expand Down Expand Up @@ -239,6 +240,19 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e
EndpointAccess: endpointAccess,
},
}
if o.AutoNode {
cluster.Spec.AutoNode = &hyperv1.AutoNode{
Provisioner: &hyperv1.ProvisionerConfig{
Name: hyperv1.ProvisionerKarpeneter,
Karpenter: &hyperv1.KarpenterConfig{
Platform: hyperv1.AWSPlatform,
AWS: &hyperv1.KarpenterAWSConfig{
RoleARN: o.iamInfo.KarpenterRoleARN,
},
},
},
}
}

if o.iamInfo.SharedIngressRoleARN != "" && o.iamInfo.SharedControlPlaneRoleARN != "" {
cluster.Spec.Platform.AWS.SharedVPC = &hyperv1.AWSSharedVPC{
Expand Down Expand Up @@ -389,6 +403,7 @@ func bindCoreOptions(opts *RawCreateOptions, flags *flag.FlagSet) {
flags.StringVar(&opts.CredentialSecretName, "secret-creds", opts.CredentialSecretName, "A Kubernetes secret with needed AWS platform credentials: sts-creds, pull-secret, and a base-domain value. The secret must exist in the supplied \"--namespace\". If a value is provided through the flag '--pull-secret', that value will override the pull-secret value in 'secret-creds'.")
flags.StringVar(&opts.IssuerURL, "oidc-issuer-url", "", "The OIDC provider issuer URL")
flags.BoolVar(&opts.MultiArch, "multi-arch", opts.MultiArch, "If true, this flag indicates the Hosted Cluster will support multi-arch NodePools and will perform additional validation checks to ensure a multi-arch release image or stream was used.")
flags.BoolVar(&opts.AutoNode, "auto-node", opts.AutoNode, "If true, this flag indicates the Hosted Cluster will support AutoNode feature.")
flags.StringVar(&opts.VPCCIDR, "vpc-cidr", opts.VPCCIDR, "The CIDR to use for the cluster VPC (mask must be 16)")
flags.BoolVar(&opts.PrivateZonesInClusterAccount, "private-zones-in-cluster-account", opts.PrivateZonesInClusterAccount, "In shared VPC infrastructure, create private hosted zones in cluster account")
flags.BoolVar(&opts.PublicOnly, "public-only", opts.PublicOnly, "If true, creates a cluster that does not have private subnets or NAT gateway and assigns public IPs to all instances.")
Expand Down Expand Up @@ -467,6 +482,7 @@ func CreateIAMOptions(awsOpts *ValidatedCreateOptions, infra *awsinfra.CreateInf
KMSKeyARN: awsOpts.EtcdKMSKeyARN,
VPCOwnerCredentialsOpts: awsOpts.VPCOwnerCredentials,
PrivateZonesInClusterAccount: awsOpts.PrivateZonesInClusterAccount,
CreateKarpenterRoleARN: awsOpts.AutoNode,
}
}

Expand Down
5 changes: 4 additions & 1 deletion cmd/infra/aws/create_iam.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ type CreateIAMOptions struct {

CredentialsSecretData *util.CredentialsSecretData

additionalIAMTags []*iam.Tag
additionalIAMTags []*iam.Tag
CreateKarpenterRoleARN bool
}

type CreateIAMOutput struct {
Expand All @@ -55,6 +56,8 @@ type CreateIAMOutput struct {

SharedIngressRoleARN string `json:"sharedIngressRoleARN,omitempty"`
SharedControlPlaneRoleARN string `json:"sharedControlPlaneRoleARN,omitempty"`

KarpenterRoleARN string `json:"karpenterRoleARN,omitempty"`
}

func NewCreateIAMCommand() *cobra.Command {
Expand Down
17 changes: 13 additions & 4 deletions cmd/infra/aws/ec2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import (
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/go-logr/logr"
"github.com/openshift/hypershift/cmd/util"

"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"k8s.io/utils/ptr"
)

const (
Expand Down Expand Up @@ -215,14 +215,20 @@ func (o *CreateInfraOptions) existingDHCPOptions(client ec2iface.EC2API) (string
}

func (o *CreateInfraOptions) CreatePrivateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) {
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB)
karpenterDiscoveryTag := []*ec2.Tag{
{
Key: ptr.To("karpenter.sh/discovery"),
Value: ptr.To(o.InfraID),
},
}
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-private-%s", o.InfraID, zone), tagNameSubnetInternalELB, karpenterDiscoveryTag)
}

func (o *CreateInfraOptions) CreatePublicSubnet(l logr.Logger, client ec2iface.EC2API, vpcID string, zone string, cidr string) (string, error) {
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB)
return o.CreateSubnet(l, client, vpcID, zone, cidr, fmt.Sprintf("%s-public-%s", o.InfraID, zone), tagNameSubnetPublicELB, nil)
}

func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string) (string, error) {
func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API, vpcID, zone, cidr, name, scopeTag string, additionalTags []*ec2.Tag) (string, error) {
subnetID, err := o.existingSubnet(client, name)
if err != nil {
return "", err
Expand All @@ -236,6 +242,9 @@ func (o *CreateInfraOptions) CreateSubnet(l logr.Logger, client ec2iface.EC2API,
Key: aws.String(scopeTag),
Value: aws.String("1"),
})
if additionalTags != nil {
tagSpec[0].Tags = append(tagSpec[0].Tags, additionalTags...)
}

result, err := client.CreateSubnet(&ec2.CreateSubnetInput{
AvailabilityZone: aws.String(zone),
Expand Down
222 changes: 222 additions & 0 deletions cmd/infra/aws/iam.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,222 @@ var (
}`,
}

// {
// "Action": [
// "*"
// ],
// "Resource": [
// "*"
// ],
// "Effect": "Allow"
// }
karpenterPolicy = policyBinding{
name: "karpenter",
serviceAccounts: []string{"system:serviceaccount:kube-system:karpenter"},
policy: `{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowScopedEC2InstanceAccessActions",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*::image/*",
"arn:*:ec2:*::snapshot/*",
"arn:*:ec2:*:*:security-group/*",
"arn:*:ec2:*:*:subnet/*"
],
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet"
]
},
{
"Sid": "AllowScopedEC2LaunchTemplateAccessActions",
"Effect": "Allow",
"Resource": "arn:*:ec2:*:*:launch-template/*",
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet"
]
},
{
"Sid": "AllowScopedEC2InstanceActionsWithTags",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:fleet/*",
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:volume/*",
"arn:*:ec2:*:*:network-interface/*",
"arn:*:ec2:*:*:launch-template/*",
"arn:*:ec2:*:*:spot-instances-request/*"
],
"Action": [
"ec2:RunInstances",
"ec2:CreateFleet",
"ec2:CreateLaunchTemplate"
],
"Condition": {
"StringLike": {
"aws:RequestTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedResourceCreationTagging",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:fleet/*",
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:volume/*",
"arn:*:ec2:*:*:network-interface/*",
"arn:*:ec2:*:*:launch-template/*",
"arn:*:ec2:*:*:spot-instances-request/*"
],
"Action": "ec2:CreateTags",
"Condition": {
"StringEquals": {
"ec2:CreateAction": [
"RunInstances",
"CreateFleet",
"CreateLaunchTemplate"
]
},
"StringLike": {
"aws:RequestTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedResourceTagging",
"Effect": "Allow",
"Resource": "arn:*:ec2:*:*:instance/*",
"Action": "ec2:CreateTags",
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowScopedDeletion",
"Effect": "Allow",
"Resource": [
"arn:*:ec2:*:*:instance/*",
"arn:*:ec2:*:*:launch-template/*"
],
"Action": [
"ec2:TerminateInstances",
"ec2:DeleteLaunchTemplate"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.sh/nodepool": "*"
}
}
},
{
"Sid": "AllowRegionalReadActions",
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:DescribeImages",
"ec2:DescribeInstances",
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeInstanceTypes",
"ec2:DescribeLaunchTemplates",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets"
]
},
{
"Sid": "AllowSSMReadActions",
"Effect": "Allow",
"Resource": "arn:*:ssm:*::parameter/aws/service/*",
"Action": "ssm:GetParameter"
},
{
"Sid": "AllowPricingReadActions",
"Effect": "Allow",
"Resource": "*",
"Action": "pricing:GetProducts"
},
{
"Sid": "AllowInterruptionQueueActions",
"Effect": "Allow",
"Resource": "*",
"Action": [
"sqs:DeleteMessage",
"sqs:GetQueueUrl",
"sqs:ReceiveMessage"
]
},
{
"Sid": "AllowPassingInstanceRole",
"Effect": "Allow",
"Resource": "arn:*:iam::*:role/*",
"Action": "iam:PassRole",
"Condition": {
"StringEquals": {
"iam:PassedToService": [
"ec2.amazonaws.com",
"ec2.amazonaws.com.cn"
]
}
}
},
{
"Sid": "AllowScopedInstanceProfileCreationActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:CreateInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowScopedInstanceProfileTagActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:TagInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*",
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowScopedInstanceProfileActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": [
"iam:AddRoleToInstanceProfile",
"iam:RemoveRoleFromInstanceProfile",
"iam:DeleteInstanceProfile"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*"
}
}
},
{
"Sid": "AllowInstanceProfileReadActions",
"Effect": "Allow",
"Resource": "arn:*:iam::*:instance-profile/*",
"Action": "iam:GetInstanceProfile"
}
]
}`,
}

nodePoolPolicy = policyBinding{
name: "node-pool",
serviceAccounts: []string{"system:serviceaccount:kube-system:capa-controller-manager"},
Expand Down Expand Up @@ -628,6 +844,12 @@ func (o *CreateIAMOptions) CreateOIDCResources(iamClient iamiface.IAMAPI, logger
&output.Roles.ControlPlaneOperatorARN: controlPlaneOperatorPolicy(o.LocalZoneID, sharedVPC),
&output.Roles.NetworkARN: cloudNetworkConfigControllerPolicy,
}

if o.CreateKarpenterRoleARN {
bindings[&output.KarpenterRoleARN] = karpenterPolicy

}

if len(o.KMSKeyARN) > 0 {
bindings[&output.KMSProviderRoleARN] = kmsProviderPolicy(o.KMSKeyARN)
}
Expand Down