From af54c62766d8a1ad0f10669f319aabd1691e0d67 Mon Sep 17 00:00:00 2001
From: Dawn Chen <dawnchen@google.com>
Date: Mon, 9 Feb 2015 13:55:36 -0800
Subject: [PATCH] Fix podstatus issue caused by docker's resource temporarily
 unavailable error

---
 pkg/kubelet/kubelet.go  | 21 ++++++++++++++-------
 pkg/master/pod_cache.go | 10 +++++-----
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
index e0d1405d27441..92255bb8337b0 100644
--- a/pkg/kubelet/kubelet.go
+++ b/pkg/kubelet/kubelet.go
@@ -1449,10 +1449,6 @@ func (kl *Kubelet) GetPodByName(namespace, name string) (*api.BoundPod, bool) {
 
 // getPhase returns the phase of a pod given its container info.
 func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
-	if info == nil {
-		return api.PodPending
-	}
-
 	running := 0
 	waiting := 0
 	stopped := 0
@@ -1481,6 +1477,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
 	}
 	switch {
 	case waiting > 0:
+		glog.V(5).Infof("pod waiting > 0, pending")
 		// One or more containers has not been started
 		return api.PodPending
 	case running > 0 && unknown == 0:
@@ -1507,6 +1504,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
 		// and in the process of restarting
 		return api.PodRunning
 	default:
+		glog.V(5).Infof("pod default case, pending")
 		return api.PodPending
 	}
 }
@@ -1555,10 +1553,19 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
 	info, err := dockertools.GetDockerPodInfo(kl.dockerClient, spec, podFullName, uid)
 
 	if err != nil {
-		glog.Infof("Query docker container info failed with error: %v", err)
-		return podStatus, err
+		// Error handling
+		glog.Infof("Query docker container info for pod %s failed with error (%v)", podFullName, err)
+		if strings.Contains(err.Error(), "resource temporarily unavailable") {
+			// Leave upstream layer to decide what to do
+			return podStatus, err
+		} else {
+			podStatus.Phase = api.PodPending
+			podStatus.Message = fmt.Sprintf("Query docker container info failed with error (%v)", err)
+			return podStatus, nil
+		}
 	}
 
+	// Assume info is ready to process
 	podStatus.Phase = getPhase(&spec, info)
 	for _, c := range spec.Containers {
 		containerStatus := info[c.Name]
@@ -1575,7 +1582,7 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
 	// TODO(dchen1107): Change Info to list from map
 	podStatus.Info = info
 
-	return podStatus, err
+	return podStatus, nil
 }
 
 // Returns logs of current machine.
diff --git a/pkg/master/pod_cache.go b/pkg/master/pod_cache.go
index 9cfe7726537c7..e1a25c90ddf23 100644
--- a/pkg/master/pod_cache.go
+++ b/pkg/master/pod_cache.go
@@ -148,7 +148,9 @@ func (p *PodCache) updatePodStatus(pod *api.Pod) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	// Map accesses must be locked.
-	p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
+	if err == nil {
+		p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
+	}
 
 	return err
 }
@@ -187,13 +189,11 @@ func (p *PodCache) computePodStatus(pod *api.Pod) (api.PodStatus, error) {
 	}
 
 	result, err := p.containerInfo.GetPodStatus(pod.Status.Host, pod.Namespace, pod.Name)
-	newStatus.HostIP = nodeStatus.HostIP
 
 	if err != nil {
-		glog.Errorf("error getting pod status: %v, setting status to unknown", err)
-		newStatus.Phase = api.PodUnknown
-		newStatus.Conditions = append(newStatus.Conditions, pod.Status.Conditions...)
+		glog.Infof("error getting pod %s status: %v, retry later", pod.Name, err)
 	} else {
+		newStatus.HostIP = nodeStatus.HostIP
 		newStatus.Info = result.Status.Info
 		newStatus.PodIP = result.Status.PodIP
 		if newStatus.Info == nil {