Garbage collect unidentified Kubernetes containers.

These containers may be caused by a change in the Kubernetes naming convention. The old containers are killed, the new ones started, but the old ones are never GC'd. This change makes Kubelet GC all Kubernetes containers, old and new. Fixes #5372.
kubernetes · Mar 12, 2015 · 5112299 · 5112299
1 parent 2939abb
commit 5112299
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 0 deletions.
diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
@@ -466,12 +466,48 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 	if err != nil {
 		return err
 	}
+
+	type unidentifiedContainer struct {
+		// Docker ID.
+		id string
+
+		// Docker container name
+		name string
+	}
+
+	unidentifiedContainers := make([]unidentifiedContainer, 0)
 	uidToIDMap := map[string][]string{}
 	for _, container := range containers {
 		_, uid, name, _ := dockertools.ParseDockerName(container.Names[0])
+		if uid == "" && name == "" {
+			unidentifiedContainers = append(unidentifiedContainers, unidentifiedContainer{
+				id:   container.ID,
+				name: container.Names[0],
+			})
+			continue
+		}
 		uidName := string(uid) + "." + name
 		uidToIDMap[uidName] = append(uidToIDMap[uidName], container.ID)
 	}
+
+	// Remove all non-running unidentified containers.
+	for _, container := range unidentifiedContainers {
+		data, err := kl.dockerClient.InspectContainer(container.id)
+		if err != nil {
+			return err
+		}
+		if data.State.Running {
+			continue
+		}
+
+		glog.Infof("Removing unidentified dead container %q with ID %q", container.name, container.id)
+		err = kl.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.id})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Evict dead containers according to our policies.
 	for _, list := range uidToIDMap {
 		if len(list) <= kl.maxContainerCount {
 			continue
@@ -480,6 +516,7 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 			return err
 		}
 	}
+
 	return nil
 }
 

diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go
@@ -1741,6 +1741,43 @@ func TestKubeletGarbageCollection(t *testing.T) {
 			},
 			expectedRemoved: []string{"1706", "1876"},
 		},
+		// Remove non-running unidentified Kubernetes containers.
+		{
+			containers: []docker.APIContainers{
+				{
+					// Unidentified Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "1876",
+				},
+				{
+					// Unidentified (non-running) Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "2309",
+				},
+				{
+					// Regular Kubernetes container.
+					Names: []string{"/k8s_POD_foo_new_.deadbeef_42"},
+					ID:    "3876",
+				},
+			},
+			containerDetails: map[string]*docker.Container{
+				"1876": {
+					State: docker.State{
+						Running: false,
+					},
+					ID:      "1876",
+					Created: time.Now(),
+				},
+				"2309": {
+					State: docker.State{
+						Running: true,
+					},
+					ID:      "2309",
+					Created: time.Now(),
+				},
+			},
+			expectedRemoved: []string{"1876"},
+		},
 	}
 	for _, test := range tests {
 		kubelet, fakeDocker, _, _ := newTestKubelet(t)