From 565de09695c6d4e64ebbf7af04251fdd89f4bfbf Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 30 Apr 2020 10:06:33 -0700 Subject: [PATCH] Updated MSI support (#1399) * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * rolling back terraform version change * Adding aks resource id to output * removing agent_pool_profile which is now considered EOL * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * Adding node_count * Adding msi_enabled var to aks-gitops module * adding system assigned identity outputs * adding system assigned identity outputs * adding system assigned identity outputs * exporting client id through data external script * Adding subscription is * Adding subscription is * removing tenant id output * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet resource id * Adding kubelet resource id * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * removing kubelet identity default * Adding vnet subnet id * version bump * creating dynamic block for sp provision * version bump * fixed aks bug * fixed aks bug * running dos2unix * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * rolling back version change * removing user identity setup and adding node resource group export * reverting flexvol changes * adding nelwine * Adding condition to support aks auto generating sp if sp client id isn't specified * reverting windows profile change * Adding sp terraform variables as optional in aks-gitops module * Adding newline * fixing node group export bug * fixing node group export bug * changing script execution permisssions * key path for gitops rename * update * Fixing SP provisioning bug * removing template for msi Co-authored-by: erikschlegel Co-authored-by: Erik Schlegel --- cluster/azure/aks-gitops/main.tf | 1 + cluster/azure/aks-gitops/outputs.tf | 24 +++++++++++ cluster/azure/aks-gitops/variables.tf | 15 +++++-- cluster/azure/aks/aks_msi_client_id_query.sh | 4 ++ cluster/azure/aks/main.tf | 38 ++++++++++++++++-- cluster/azure/aks/outputs.tf | 24 +++++++++++ cluster/azure/aks/variables.tf | 42 ++++++++++++-------- 7 files changed, 125 insertions(+), 23 deletions(-) create mode 100755 cluster/azure/aks/aks_msi_client_id_query.sh diff --git a/cluster/azure/aks-gitops/main.tf b/cluster/azure/aks-gitops/main.tf index 7a881bf66..5777103ac 100644 --- a/cluster/azure/aks-gitops/main.tf +++ b/cluster/azure/aks-gitops/main.tf @@ -12,6 +12,7 @@ module "aks" { dns_prefix = var.dns_prefix vnet_subnet_id = var.vnet_subnet_id ssh_public_key = var.ssh_public_key + msi_enabled = var.msi_enabled service_principal_id = var.service_principal_id service_principal_secret = var.service_principal_secret service_cidr = var.service_cidr diff --git a/cluster/azure/aks-gitops/outputs.tf b/cluster/azure/aks-gitops/outputs.tf index 671586ad8..636f02919 100644 --- a/cluster/azure/aks-gitops/outputs.tf +++ b/cluster/azure/aks-gitops/outputs.tf @@ -5,3 +5,27 @@ output "kubeconfig_done" { output "aks_flux_kubediff_done" { value = "${module.aks.kubeconfig_done}_${module.flux.flux_done}_${module.kubediff.kubediff_done}" } + +output "aks_resource_id" { + value = module.aks.resource_id +} + +output "msi_client_id" { + value = module.aks.msi_client_id +} + +output "kubelet_client_id" { + value = module.aks.kubelet_client_id +} + +output "kubelet_id" { + value = module.aks.kubelet_id +} + +output "kubelet_resource_id" { + value = module.aks.kubelet_resource_id +} + +output "node_resource_group" { + value = module.aks.node_resource_group +} diff --git a/cluster/azure/aks-gitops/variables.tf b/cluster/azure/aks-gitops/variables.tf index 39c57e0ba..85475b1d8 100644 --- a/cluster/azure/aks-gitops/variables.tf +++ b/cluster/azure/aks-gitops/variables.tf @@ -19,6 +19,11 @@ variable "cluster_name" { type = string } +variable "msi_enabled" { + type = bool + default = false +} + variable "dns_prefix" { type = string } @@ -67,20 +72,22 @@ variable "resource_group_name" { type = string } -variable "service_principal_id" { +variable "ssh_public_key" { type = string } -variable "service_principal_secret" { +variable "vnet_subnet_id" { type = string } -variable "ssh_public_key" { +variable "service_principal_id" { type = string + default = "" } -variable "vnet_subnet_id" { +variable "service_principal_secret" { type = string + default = "" } variable "service_cidr" { diff --git a/cluster/azure/aks/aks_msi_client_id_query.sh b/cluster/azure/aks/aks_msi_client_id_query.sh new file mode 100755 index 000000000..0fe9aaf04 --- /dev/null +++ b/cluster/azure/aks/aks_msi_client_id_query.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +az aks show -n $1 -g $2 --subscription $3 --query "{kubelet_client_id:identityProfile.kubeletidentity.objectId,msi_client_id:identity.principalId,kubelet_id:identityProfile.kubeletidentity.resourceId,kubelet_resource_id:identityProfile.kubeletidentity.resourceId,node_resource_group:nodeResourceGroup}" \ No newline at end of file diff --git a/cluster/azure/aks/main.tf b/cluster/azure/aks/main.tf index 9c9a8335e..07b3c4f5b 100644 --- a/cluster/azure/aks/main.tf +++ b/cluster/azure/aks/main.tf @@ -1,7 +1,13 @@ +locals { + msi_identity_type = "SystemAssigned" +} + data "azurerm_resource_group" "cluster" { name = var.resource_group_name } +data "azurerm_subscription" "current" {} + resource "random_id" "workspace" { keepers = { group_name = data.azurerm_resource_group.cluster.name @@ -73,9 +79,15 @@ resource "azurerm_kubernetes_cluster" "cluster" { enabled = true } - service_principal { - client_id = var.service_principal_id - client_secret = var.service_principal_secret + dynamic "service_principal" { + for_each = !var.msi_enabled && var.service_principal_id != "" ? [{ + client_id = var.service_principal_id + client_secret = var.service_principal_secret + }] : [] + content { + client_id = service_principal.value.client_id + client_secret = service_principal.value.client_secret + } } addon_profile { @@ -84,4 +96,24 @@ resource "azurerm_kubernetes_cluster" "cluster" { log_analytics_workspace_id = azurerm_log_analytics_workspace.workspace.id } } + + # This dynamic block enables managed service identity for the cluster + # in the case that the following holds true: + # 1: the msi_enabled input variable is set to true + dynamic "identity" { + for_each = var.msi_enabled ? [local.msi_identity_type] : [] + content { + type = identity.value + } + } +} + +data "external" "msi_object_id" { + depends_on = [azurerm_kubernetes_cluster.cluster] + program = [ + "${path.module}/aks_msi_client_id_query.sh", + var.cluster_name, + data.azurerm_resource_group.cluster.name, + data.azurerm_subscription.current.subscription_id + ] } diff --git a/cluster/azure/aks/outputs.tf b/cluster/azure/aks/outputs.tf index 1f9d8da74..c8ccbe68f 100644 --- a/cluster/azure/aks/outputs.tf +++ b/cluster/azure/aks/outputs.tf @@ -11,3 +11,27 @@ output "kube_config" { output "kubeconfig_done" { value = join("", local_file.cluster_credentials.*.id) } + +output "resource_id" { + value = azurerm_kubernetes_cluster.cluster.id +} + +output "msi_client_id" { + value = data.external.msi_object_id.result.msi_client_id +} + +output "kubelet_client_id" { + value = data.external.msi_object_id.result.kubelet_client_id +} + +output "kubelet_id" { + value = data.external.msi_object_id.result.kubelet_id +} + +output "node_resource_group" { + value = data.external.msi_object_id.result.node_resource_group +} + +output "kubelet_resource_id" { + value = data.external.msi_object_id.result.kubelet_resource_id +} diff --git a/cluster/azure/aks/variables.tf b/cluster/azure/aks/variables.tf index 70291ae90..9dacc89d9 100644 --- a/cluster/azure/aks/variables.tf +++ b/cluster/azure/aks/variables.tf @@ -2,21 +2,40 @@ variable "resource_group_name" { type = string } -variable "cluster_name" { - type = string - default = "bedrockaks" +variable "dns_prefix" { + type = string } -variable "dns_prefix" { +variable "kubernetes_version" { + type = string +} + +variable "ssh_public_key" { + type = string +} + +variable "vnet_subnet_id" { type = string } variable "service_principal_id" { type = string + default = "" } variable "service_principal_secret" { type = string + default = "" +} + +variable "msi_enabled" { + type = bool + default = false +} + +variable "cluster_name" { + type = string + default = "bedrockaks" } variable "agent_vm_count" { @@ -29,28 +48,16 @@ variable "agent_vm_size" { default = "Standard_D2s_v3" } -variable "kubernetes_version" { - type = string -} - variable "admin_user" { type = string default = "k8sadmin" } -variable "ssh_public_key" { - type = string -} - variable "output_directory" { type = string default = "./output" } -variable "vnet_subnet_id" { - type = string -} - variable "enable_virtual_node_addon" { type = string default = "false" @@ -81,16 +88,19 @@ variable "dns_ip" { } variable "docker_cidr" { + type = string default = "172.17.0.1/16" description = "IP address (in CIDR notation) used as the Docker bridge IP address on nodes. Default of 172.17.0.1/16." } variable "network_plugin" { default = "azure" + type = string description = "Network plugin used by AKS. Either azure or kubenet." } variable "network_policy" { default = "azure" + type = string description = "Network policy to be used with Azure CNI. Either azure or calico." }