Catch kubelet-master hostname mismatch during health check

During the kubelet's /healthz responce check to see if the hostname used by the master matches the hostname the kubelet knows itself by. If not fail the health check. Signed-off-by: Sami Wagiaalla <swagiaal@redhat.com>
justinsb · Feb 26, 2015 · 9150cb9 · 9150cb9
1 parent eedd9ec
commit 9150cb9
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 0 deletions.
diff --git a/hack/test-cmd.sh b/hack/test-cmd.sh
@@ -57,6 +57,7 @@ kube::log::status "Starting kubelet in masterless mode"
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
   --docker_endpoint="fake://" \
+  --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
   --port="$KUBELET_PORT" 1>&2 &
 KUBELET_PID=$!

diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
@@ -1545,6 +1545,11 @@ func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName, tail stri
 	return dockertools.GetKubeletDockerContainerLogs(kl.dockerClient, dockerContainerID, tail, follow, stdout, stderr)
 }
 
+// GetHostname Returns the hostname as the kubelet sees it.
+func (kl *Kubelet) GetHostname() string {
+	return kl.hostname
+}
+
 // GetBoundPods returns all pods bound to the kubelet and their spec
 func (kl *Kubelet) GetBoundPods() ([]api.BoundPod, error) {
 	kl.podLock.RLock()

diff --git a/pkg/kubelet/server.go b/pkg/kubelet/server.go
@@ -78,6 +78,7 @@ type HostInterface interface {
 	ServeLogs(w http.ResponseWriter, req *http.Request)
 	PortForward(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error
 	StreamingConnectionIdleTimeout() time.Duration
+	GetHostname() string
 }
 
 // NewServer initializes and configures a kubelet.Server object to handle HTTP requests.
@@ -148,6 +149,25 @@ func (s *Server) handleHealthz(w http.ResponseWriter, req *http.Request) {
 		s.error(w, errors.New("Docker version is too old ("+version+")"))
 		return
 	}
+
+	masterHostname, _, err := net.SplitHostPort(req.Host)
+	if err != nil {
+		if !strings.Contains(req.Host, ":") {
+			masterHostname = req.Host
+		} else {
+			msg := fmt.Sprintf("Could not parse hostname from http request: %v", err)
+			s.error(w, errors.New(msg))
+			return
+		}
+	}
+
+	// Check that the hostname known by the master matches the hostname
+	// the kubelet knows
+	hostname := s.host.GetHostname()
+	if masterHostname != hostname {
+		s.error(w, errors.New("Kubelet hostname \""+hostname+"\" does not match the hostname expected by the master \""+masterHostname+"\""))
+		return
+	}
 	w.Write([]byte("ok"))
 }
 

diff --git a/pkg/kubelet/server_test.go b/pkg/kubelet/server_test.go
@@ -51,6 +51,7 @@ type fakeKubelet struct {
 	portForwardFunc                    func(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error
 	containerLogsFunc                  func(podFullName, containerName, tail string, follow bool, stdout, stderr io.Writer) error
 	streamingConnectionIdleTimeoutFunc func() time.Duration
+	hostnameFunc                       func() string
 }
 
 func (fk *fakeKubelet) GetPodByName(namespace, name string) (*api.BoundPod, bool) {
@@ -89,6 +90,10 @@ func (fk *fakeKubelet) GetKubeletContainerLogs(podFullName, containerName, tail
 	return fk.containerLogsFunc(podFullName, containerName, tail, follow, stdout, stderr)
 }
 
+func (fk *fakeKubelet) GetHostname() string {
+	return fk.hostnameFunc()
+}
+
 func (fk *fakeKubelet) RunInContainer(podFullName string, uid types.UID, containerName string, cmd []string) ([]byte, error) {
 	return fk.runFunc(podFullName, uid, containerName, cmd)
 }
@@ -441,6 +446,63 @@ func TestPodsInfo(t *testing.T) {
 	}
 }
 
+func TestHealthCheck(t *testing.T) {
+	fw := newServerTest()
+	fw.fakeKubelet.dockerVersionFunc = func() ([]uint, error) {
+		return []uint{1, 15}, nil
+	}
+	fw.fakeKubelet.hostnameFunc = func() string {
+		return "127.0.0.1"
+	}
+
+	// Test with correct hostname, Docker version
+	resp, err := http.Get(fw.testHTTPServer.URL + "/healthz")
+	if err != nil {
+		t.Fatalf("Got error GETing: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("expected status code %d, got %d", http.StatusOK, resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		// copying the response body did not work
+		t.Fatalf("Cannot copy resp: %#v", err)
+	}
+	result := string(body)
+	if !strings.Contains(result, "ok") {
+		t.Errorf("expected body contains %s, got %d", "ok", result)
+	}
+
+	//Test with incorrect hostname
+	fw.fakeKubelet.hostnameFunc = func() string {
+		return "fake"
+	}
+	resp, err = http.Get(fw.testHTTPServer.URL + "/healthz")
+	if err != nil {
+		t.Fatalf("Got error GETing: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusInternalServerError {
+		t.Errorf("expected status code %d, got %d", http.StatusOK, resp.StatusCode)
+	}
+
+	//Test with old docker version
+	fw.fakeKubelet.dockerVersionFunc = func() ([]uint, error) {
+		return []uint{1, 1}, nil
+	}
+
+	resp, err = http.Get(fw.testHTTPServer.URL + "/healthz")
+	if err != nil {
+		t.Fatalf("Got error GETing: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusInternalServerError {
+		t.Errorf("expected status code %d, got %d", http.StatusOK, resp.StatusCode)
+	}
+
+}
+
 func setPodByNameFunc(fw *serverTestFramework, namespace, pod, container string) {
 	fw.fakeKubelet.podByNameFunc = func(namespace, name string) (*api.BoundPod, bool) {
 		return &api.BoundPod{