kubernetes · k8s-github-robot · May 31, 2017 · Feb 17, 2017 · liggitt · May 27, 2017
diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go
@@ -288,7 +288,6 @@ func initKubeletConfigSync(s *options.KubeletServer) (*componentconfig.KubeletCo
 func Run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) error {
 	if err := run(s, kubeDeps); err != nil {
 		return fmt.Errorf("failed to run Kubelet: %v", err)
-
 	}
 	return nil
 }
@@ -623,7 +622,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName
 // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
 // certificate and key file are generated. Returns a configured server.TLSOptions object.
 func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) {
-	if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
 		kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
 		kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
 

diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
@@ -89,6 +89,14 @@ const (
 	// to take advantage of NoExecute Taints and Tolerations.
 	TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions"
 
+	// owner: @jcbsmpsn
+	// alpha: v1.7
+	//
+	// Gets a server certificate for the kubelet from the Certificate Signing
+	// Request API instead of generating one self signed and auto rotates the
+	// certificate as expiration approaches.
+	RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate"
+
 	// owner: @msau
 	// alpha: v1.7
 	//
@@ -113,6 +121,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	AffinityInAnnotations:                       {Default: false, PreRelease: utilfeature.Alpha},
 	Accelerators:                                {Default: false, PreRelease: utilfeature.Alpha},
 	TaintBasedEvictions:                         {Default: false, PreRelease: utilfeature.Alpha},
+	RotateKubeletServerCertificate:              {Default: false, PreRelease: utilfeature.Alpha},
 	PersistentLocalVolumes:                      {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed

diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD
@@ -42,16 +42,19 @@ go_library(
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/api/v1/resource:go_default_library",
         "//pkg/api/v1/validation:go_default_library",
+        "//pkg/apis/certificates/v1beta1:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/capabilities:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library",
         "//pkg/client/listers/core/v1:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/fieldpath:go_default_library",
         "//pkg/kubelet/apis/cri:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
+        "//pkg/kubelet/certificate:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/container:go_default_library",

diff --git a/pkg/kubelet/certificate/certificate_store.go b/pkg/kubelet/certificate/certificate_store.go
@@ -195,6 +195,9 @@ func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) {
 	ts := time.Now().Format("2006-01-02-15-04-05")
 	pemFilename := s.filename(ts)
 
+	if err := os.MkdirAll(s.certDirectory, 0755); err != nil {
+		return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err)
+	}
 	certPath := filepath.Join(s.certDirectory, pemFilename)
 
 	f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)

diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
@@ -17,6 +17,9 @@ limitations under the License.
 package kubelet
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"fmt"
 	"net"
 	"net/http"
@@ -51,14 +54,17 @@ import (
 	"k8s.io/client-go/util/integer"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
+	certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	clientcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1"
 	corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/features"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -304,6 +310,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 	hostname := nodeutil.GetHostname(hostnameOverride)
 	// Query the cloud provider for our node name, default to hostname
 	nodeName := types.NodeName(hostname)
+	cloudIPs := []net.IP{}
+	cloudNames := []string{}
 	if kubeDeps.Cloud != nil {
 		var err error
 		instances, ok := kubeDeps.Cloud.Instances()
@@ -317,6 +325,25 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		}
 
 		glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
+
+		if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+			nodeAddresses, err := instances.NodeAddresses(nodeName)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err)
+			}
+			for _, nodeAddress := range nodeAddresses {
+				switch nodeAddress.Type {
+				case v1.NodeExternalIP, v1.NodeInternalIP:
+					ip := net.ParseIP(nodeAddress.Address)
+					if ip != nil && !ip.IsLoopback() {
+						cloudIPs = append(cloudIPs, ip)
+					}
+				case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName:
+					cloudNames = append(cloudNames, nodeAddress.Address)
+				}
+			}
+		}
+
 	}
 
 	if kubeDeps.PodConfig == nil {
@@ -654,6 +681,33 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 
 	klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kubeDeps.TLSOptions != nil {
+		var ips []net.IP
+		cfgAddress := net.ParseIP(kubeCfg.Address)
+		if cfgAddress == nil || cfgAddress.IsUnspecified() {
+			if localIPs, err := allLocalIPsWithoutLoopback(); err != nil {
+				return nil, err
+			} else {
+				ips = localIPs
+			}
+		} else {
+			ips = []net.IP{cfgAddress}
+		}
+		ips = append(ips, cloudIPs...)
+		names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
+		klet.serverCertificateManager, err = initializeServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
+		}
+		kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := klet.serverCertificateManager.Current()
+			if cert == nil {
+				return nil, fmt.Errorf("no certificate available")
+			}
+			return cert, nil
+		}
+	}
+
 	klet.probeManager = prober.NewManager(
 		klet.statusManager,
 		klet.livenessManager,
@@ -865,6 +919,9 @@ type Kubelet struct {
 	// Cached MachineInfo returned by cadvisor.
 	machineInfo *cadvisorapi.MachineInfo
 
+	// Handles certificate rotations.
+	serverCertificateManager certificate.Manager
+
 	// Syncs pods statuses with apiserver; also used as a cache of statuses.
 	statusManager status.Manager
 
@@ -1037,6 +1094,62 @@ type Kubelet struct {
 	dockerLegacyService dockershim.DockerLegacyService
 }
 
+func initializeServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (certificate.Manager, error) {
+	var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
+	if kubeClient != nil && kubeClient.Certificates() != nil {
+		certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
+	}
+	certificateStore, err := certificate.NewFileStore(
+		"kubelet-server",
+		kubeCfg.CertDirectory,
+		kubeCfg.CertDirectory,
+		kubeCfg.TLSCertFile,
+		kubeCfg.TLSPrivateKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize certificate store: %v", err)
+	}
+	return certificate.NewManager(&certificate.Config{
+		CertificateSigningRequestClient: certSigningRequestClient,
+		Template: &x509.CertificateRequest{
+			Subject: pkix.Name{
+				CommonName:   string(nodeName),
+				Organization: []string{"system:nodes"},
+			},
+			DNSNames:    hostnames,
+			IPAddresses: ips,
+		},
+		Usages: []certificates.KeyUsage{
+			certificates.UsageKeyEncipherment,
+			certificates.UsageServerAuth,
+			certificates.UsageSigning,
+		},
+		CertificateStore: certificateStore,
+	})
+}
+
+func allLocalIPsWithoutLoopback() ([]net.IP, error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("could not list network interfaces: %v", err)
+	}
+	var ips []net.IP
+	for _, i := range interfaces {
+		addresses, err := i.Addrs()
+		if err != nil {
+			return nil, fmt.Errorf("could not list the addresses for network interface %v: %v\n", i, err)
+		}
+		for _, address := range addresses {
+			switch v := address.(type) {
+			case *net.IPNet:
+				if !v.IP.IsLoopback() {
+					ips = append(ips, v.IP)
+				}
+			}
+		}
+	}
+	return ips, nil
+}
+
 // setupDataDirs creates:
 // 1.  the root directory
 // 2.  the pods directory
@@ -1100,25 +1213,30 @@ func (kl *Kubelet) StartGarbageCollection() {
 // initializeModules will initialize internal modules that do not require the container runtime to be up.
 // Note that the modules here must not depend on modules that are not initialized here.
 func (kl *Kubelet) initializeModules() error {
-	// Step 1: Prometheus metrics.
+	// Prometheus metrics.
 	metrics.Register(kl.runtimeCache)
 
-	// Step 2: Setup filesystem directories.
+	// Setup filesystem directories.
 	if err := kl.setupDataDirs(); err != nil {
 		return err
 	}
 
-	// Step 3: If the container logs directory does not exist, create it.
+	// If the container logs directory does not exist, create it.
 	if _, err := os.Stat(ContainerLogsDir); err != nil {
 		if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
 			glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
 		}
 	}
 
-	// Step 4: Start the image manager.
+	// Start the image manager.
 	kl.imageManager.Start()
 
-	// Step 5: Start container manager.
+	// Start the certificate manager.
+	if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+		kl.serverCertificateManager.Start()
+	}
+
+	// Start container manager.
 	node, err := kl.getNodeAnyWay()
 	if err != nil {
 		return fmt.Errorf("Kubelet failed to get node info: %v", err)
@@ -1128,17 +1246,17 @@ func (kl *Kubelet) initializeModules() error {
 		return fmt.Errorf("Failed to start ContainerManager %v", err)
 	}
 
-	// Step 6: Start out of memory watcher.
+	// Start out of memory watcher.
 	if err := kl.oomWatcher.Start(kl.nodeRef); err != nil {
 		return fmt.Errorf("Failed to start OOM watcher %v", err)
 	}
 
-	// Step 7: Initialize GPUs
+	// Initialize GPUs
 	if err := kl.gpuManager.Start(); err != nil {
 		glog.Errorf("Failed to start gpuManager %v", err)
 	}
 
-	// Step 8: Start resource analyzer
+	// Start resource analyzer
 	kl.resourceAnalyzer.Start()
 
 	return nil

diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go
@@ -134,6 +134,9 @@ func ListenAndServeKubeletServer(
 	}
 	if tlsOptions != nil {
 		s.TLSConfig = tlsOptions.Config
+		// Passing empty strings as the cert and key files means no
+		// cert/keys are specified and GetCertificate in the TLSConfig
+		// should be called instead.
 		glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile))
 	} else {
 		glog.Fatal(s.ListenAndServe())