Merge pull request kubernetes#6243 from roberthbailey/kubelet-ssl

Configure the kubelet to use HTTPS
duda · Apr 1, 2015 · 92b6f49 · 92b6f49
2 parents f7f1337 + dd9ec46
commit 92b6f49
Show file tree

Hide file tree

Showing 9 changed files with 126 additions and 71 deletions.
diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go
@@ -99,7 +99,7 @@ func NewAPIServer() *APIServer {
 		RuntimeConfig: make(util.ConfigurationMap),
 		KubeletConfig: client.KubeletConfig{
 			Port:        10250,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go
@@ -83,7 +83,7 @@ func NewCMServer() *CMServer {
 		SyncNodeStatus:          false,
 		KubeletConfig: client.KubeletConfig{
 			Port:        ports.KubeletPort,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go
@@ -18,10 +18,12 @@ limitations under the License.
 package app
 
 import (
+	"crypto/tls"
 	"fmt"
 	"math/rand"
 	"net"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -88,6 +90,9 @@ type KubeletServer struct {
 	NetworkPluginName              string
 	CloudProvider                  string
 	CloudConfigFile                string
+	TLSCertFile                    string
+	TLSPrivateKeyFile              string
+	CertDirectory                  string
 }
 
 // NewKubeletServer will create a new KubeletServer with default values.
@@ -115,6 +120,7 @@ func NewKubeletServer() *KubeletServer {
 		ImageGCLowThresholdPercent:  80,
 		NetworkPluginName:           "",
 		HostNetworkSources:          kubelet.FileSource,
+		CertDirectory:               "/var/run/kubernetes",
 	}
 }
 
@@ -128,6 +134,12 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.EnableServer, "enable_server", s.EnableServer, "Enable the info server")
 	fs.Var(&s.Address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.UintVar(&s.Port, "port", s.Port, "The port for the info server to serve on")
+	fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+
+		"File containing x509 Certificate for HTTPS.  (CA cert, if any, concatenated after server cert). "+
+		"If --tls_cert_file and --tls_private_key_file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to the directory passed to --cert_dir.")
+	fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.")
+	fs.StringVar(&s.CertDirectory, "cert_dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes)")
 	fs.StringVar(&s.HostnameOverride, "hostname_override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
 	fs.StringVar(&s.PodInfraContainerImage, "pod_infra_container_image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
 	fs.StringVar(&s.DockerEndpoint, "docker_endpoint", s.DockerEndpoint, "If non-empty, use this for the docker endpoint to communicate with")
@@ -194,6 +206,26 @@ func (s *KubeletServer) Run(_ []string) error {
 	if err != nil {
 		return err
 	}
+
+	if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
+		s.TLSCertFile = path.Join(s.CertDirectory, "kubelet.crt")
+		s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "kubelet.key")
+		if err := util.GenerateSelfSignedCert(util.GetHostname(s.HostnameOverride), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
+			glog.Fatalf("Unable to generate self signed cert: %v", err)
+		}
+		glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
+	}
+	tlsOptions := &kubelet.TLSOptions{
+		Config: &tls.Config{
+			// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability).
+			MinVersion: tls.VersionTLS10,
+			// Populate PeerCertificates in requests, but don't yet reject connections without certificates.
+			ClientAuth: tls.RequestClientCert,
+		},
+		CertFile: s.TLSCertFile,
+		KeyFile:  s.TLSPrivateKeyFile,
+	}
+
 	kcfg := KubeletConfig{
 		Address:                        s.Address,
 		AllowPrivileged:                s.AllowPrivileged,
@@ -225,6 +257,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		NetworkPlugins:                 ProbeNetworkPlugins(),
 		NetworkPluginName:              s.NetworkPluginName,
 		StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
+		TLSOptions:                     tlsOptions,
 		ImageGCPolicy:                  imageGCPolicy,
 		Cloud:                          cloud,
 	}

diff --git a/hack/test-cmd.sh b/hack/test-cmd.sh
@@ -48,6 +48,7 @@ ETCD_PORT=${ETCD_PORT:-4001}
 API_PORT=${API_PORT:-8080}
 API_HOST=${API_HOST:-127.0.0.1}
 KUBELET_PORT=${KUBELET_PORT:-10250}
+KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
 CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
 
 # Check kubectl
@@ -58,27 +59,31 @@ kube::log::status "Starting kubelet in masterless mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: "
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: "
 kill ${KUBELET_PID} 1>&2 2>/dev/null
 
 kube::log::status "Starting kubelet in masterful mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
   --api_servers="${API_HOST}:${API_PORT}" \
   --auth_path="${KUBE_ROOT}/hack/.test-cmd-auth" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
 
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: "
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: "
 
 # Start kube-apiserver
 kube::log::status "Starting kube-apiserver"

diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
@@ -139,14 +139,7 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
 
 // TODO: Convert to go-restful
 func InstallValidator(mux Mux, servers func() map[string]Server) {
-	validator, err := NewValidator(servers)
-	if err != nil {
-		glog.Errorf("failed to set up validator: %v", err)
-		return
-	}
-	if validator != nil {
-		mux.Handle("/validate", validator)
-	}
+	mux.Handle("/validate", NewValidator(servers))
 }
 
 // TODO: document all handlers

diff --git a/pkg/apiserver/validator.go b/pkg/apiserver/validator.go
@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -34,21 +35,26 @@ type httpGet interface {
 }
 
 type Server struct {
-	Addr string
-	Port int
-	Path string
+	Addr        string
+	Port        int
+	Path        string
+	EnableHTTPS bool
 }
 
 // validator is responsible for validating the cluster and serving
 type validator struct {
 	// a list of servers to health check
 	servers func() map[string]Server
-	client  httpGet
+	rt      http.RoundTripper
 }
 
 // TODO: can this use pkg/probe/http
 func (s *Server) check(client httpGet) (probe.Result, string, error) {
-	resp, err := client.Get("http://" + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
+	scheme := "http://"
+	if s.EnableHTTPS {
+		scheme = "https://"
+	}
+	resp, err := client.Get(scheme + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
 	if err != nil {
 		return probe.Unknown, "", err
 	}
@@ -81,7 +87,22 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	reply := []ServerStatus{}
 	for name, server := range v.servers() {
-		status, msg, err := server.check(v.client)
+		transport := v.rt
+		if server.EnableHTTPS {
+			// TODO(roberthbailey): The servers that use HTTPS are currently the
+			// kubelets, and we should be using a standard kubelet client library
+			// to talk to them rather than a separate http client.
+			transport = &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				Dial: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).Dial,
+				TLSHandshakeTimeout: 10 * time.Second,
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
+			}
+		}
+		status, msg, err := server.check(&http.Client{Transport: transport})
 		var errorMsg string
 		if err != nil {
 			errorMsg = err.Error()
@@ -103,30 +124,6 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 // NewValidator creates a validator for a set of servers.
-func NewValidator(servers func() map[string]Server) (http.Handler, error) {
-	return &validator{
-		servers: servers,
-		client:  &http.Client{},
-	}, nil
-}
-
-func makeTestValidator(servers map[string]string, get httpGet) (http.Handler, error) {
-	result := map[string]Server{}
-	for name, value := range servers {
-		host, port, err := net.SplitHostPort(value)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
-		}
-		val, err := strconv.Atoi(port)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
-		}
-		result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
-	}
-
-	v, e := NewValidator(func() map[string]Server { return result })
-	if e == nil {
-		v.(*validator).client = get
-	}
-	return v, e
+func NewValidator(servers func() map[string]Server) http.Handler {
+	return &validator{servers: servers, rt: http.DefaultTransport}
 }
diff --git a/pkg/apiserver/validator_test.go b/pkg/apiserver/validator_test.go
@@ -21,35 +21,27 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
-type fakeHttpGet struct {
+type fakeRoundTripper struct {
 	err  error
 	resp *http.Response
 	url  string
 }
 
-func (f *fakeHttpGet) Get(url string) (*http.Response, error) {
-	f.url = url
+func (f *fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	f.url = req.URL.String()
 	return f.resp, f.err
 }
 
-func makeFake(data string, statusCode int, err error) *fakeHttpGet {
-	return &fakeHttpGet{
-		err: err,
-		resp: &http.Response{
-			Body:       ioutil.NopCloser(bytes.NewBufferString(data)),
-			StatusCode: statusCode,
-		},
-	}
-}
-
 func TestValidate(t *testing.T) {
 	tests := []struct {
 		err            error
@@ -66,11 +58,18 @@ func TestValidate(t *testing.T) {
 	s := Server{Addr: "foo.com", Port: 8080, Path: "/healthz"}
 
 	for _, test := range tests {
-		fake := makeFake(test.data, test.code, test.err)
+		fakeRT := &fakeRoundTripper{
+			err: test.err,
+			resp: &http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(test.data)),
+				StatusCode: test.code,
+			},
+		}
+		fake := &http.Client{Transport: fakeRT}
 		status, data, err := s.check(fake)
 		expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
-		if fake.url != expect {
-			t.Errorf("expected %s, got %s", expect, fake.url)
+		if fakeRT.url != expect {
+			t.Errorf("expected %s, got %s", expect, fakeRT.url)
 		}
 		if test.expectErr && err == nil {
 			t.Errorf("unexpected non-error")
@@ -87,8 +86,30 @@ func TestValidate(t *testing.T) {
 	}
 }
 
+func makeTestValidator(servers map[string]string, rt http.RoundTripper) (http.Handler, error) {
+	result := map[string]Server{}
+	for name, value := range servers {
+		host, port, err := net.SplitHostPort(value)
+		if err != nil {
+			return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
+		}
+		val, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
+		}
+		result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
+	}
+
+	return &validator{servers: func() map[string]Server { return result }, rt: rt}, nil
+}
+
 func TestValidator(t *testing.T) {
-	fake := makeFake("foo", 200, nil)
+	fake := &fakeRoundTripper{
+		resp: &http.Response{
+			Body:       ioutil.NopCloser(bytes.NewBufferString("foo")),
+			StatusCode: 200,
+		},
+	}
 	validator, err := makeTestValidator(map[string]string{
 		"foo": "foo.com:80",
 		"bar": "bar.com:8080",
@@ -101,7 +122,6 @@ func TestValidator(t *testing.T) {
 	defer testServer.Close()
 
 	resp, err := http.Get(testServer.URL + "/validatez")
-
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -113,13 +133,15 @@ func TestValidator(t *testing.T) {
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
-	status := []ServerStatus{}
-	err = json.Unmarshal(data, &status)
-	if err != nil {
+	var status []ServerStatus
+	if err := json.Unmarshal(data, &status); err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
 	components := util.StringSet{}
 	for _, s := range status {
+		if s.Err != "nil" {
+			t.Errorf("Component %v is unhealthy: %v", s.Component, s.Err)
+		}
 		components.Insert(s.Component)
 	}
 	if len(status) != 2 || !components.Has("foo") || !components.Has("bar") {

diff --git a/pkg/client/kubelet.go b/pkg/client/kubelet.go
@@ -75,9 +75,14 @@ type HTTPKubeletClient struct {
 func NewKubeletClient(config *KubeletConfig) (KubeletClient, error) {
 	transport := http.DefaultTransport
 
-	tlsConfig, err := TLSConfigFor(&Config{
-		TLSClientConfig: config.TLSClientConfig,
-	})
+	cfg := &Config{TLSClientConfig: config.TLSClientConfig}
+	if config.EnableHttps {
+		hasCA := len(config.CAFile) > 0 || len(config.CAData) > 0
+		if !hasCA {
+			cfg.Insecure = true
+		}
+	}
+	tlsConfig, err := TLSConfigFor(cfg)
 	if err != nil {
 		return nil, err
 	}