Merge pull request #25982 from derekwaynecarr/fix_stats

Automatic merge from submit-queue Fix system container detection in kubelet on systemd ```release-note Fix system container detection in kubelet on systemd. This fixed environments where CPU and Memory Accounting were not enabled on the unit that launched the kubelet or docker from reporting the root cgroup when monitoring usage stats for those components. ``` Fixes #25909 /cc @kubernetes/sig-node @kubernetes/rh-cluster-infra @vishh @dchen1107
kubernetes · May 28, 2016 · c730198 · c730198
2 parents 04bdd37 + 5a8851d
commit c730198
Showing 1 changed file with 35 additions and 5 deletions.
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
@@ -536,19 +536,49 @@ func ensureProcessInContainer(pid int, oomScoreAdj int, manager *fs.Manager) err
 	return utilerrors.NewAggregate(errs)
 }
 
-// Gets the (CPU) container the specified pid is in.
+// getContainer returns the cgroup associated with the specified pid.
+// It enforces a unified hierarchy for memory and cpu cgroups.
+// On systemd environments, it uses the name=systemd cgroup for the specified pid.
 func getContainer(pid int) (string, error) {
 	cgs, err := cgroups.ParseCgroupFile(fmt.Sprintf("/proc/%d/cgroup", pid))
 	if err != nil {
 		return "", err
 	}
 
-	cg, ok := cgs["cpu"]
-	if ok {
-		return cg, nil
+	cpu, found := cgs["cpu"]
+	if !found {
+		return "", cgroups.NewNotFoundError("cpu")
+	}
+	memory, found := cgs["memory"]
+	if !found {
+		return "", cgroups.NewNotFoundError("memory")
+	}
+
+	// since we use this container for accounting, we need to ensure its a unified hierarchy.
+	if cpu != memory {
+		return "", fmt.Errorf("cpu and memory cgroup hierarchy not unified.  cpu: %s, memory: %s", cpu, memory)
+	}
+
+	// on systemd, every pid is in a unified cgroup hierarchy (name=systemd as seen in systemd-cgls)
+	// cpu and memory accounting is off by default, users may choose to enable it per unit or globally.
+	// users could enable CPU and memory accounting globally via /etc/systemd/system.conf (DefaultCPUAccounting=true DefaultMemoryAccounting=true).
+	// users could also enable CPU and memory accounting per unit via CPUAccounting=true and MemoryAccounting=true
+	// we only warn if accounting is not enabled for CPU or memory so as to not break local development flows where kubelet is launched in a terminal.
+	// for example, the cgroup for the user session will be something like /user.slice/user-X.slice/session-X.scope, but the cpu and memory
+	// cgroup will be the closest ancestor where accounting is performed (most likely /) on systems that launch docker containers.
+	// as a result, on those systems, you will not get cpu or memory accounting statistics for kubelet.
+	// in addition, you would not get memory or cpu accounting for the runtime unless accounting was enabled on its unit (or globally).
+	if systemd, found := cgs["name=systemd"]; found {
+		if systemd != cpu {
+			glog.Warningf("CPUAccounting not enabled for pid: %d", pid)
+		}
+		if systemd != memory {
+			glog.Warningf("MemoryAccounting not enabled for pid: %d", pid)
+		}
+		return systemd, nil
 	}
 
-	return "", cgroups.NewNotFoundError("cpu")
+	return cpu, nil
 }
 
 // Ensures the system container is created and all non-kernel threads and process 1