Merge pull request #23595 from vishh/image-accounting

Automatic merge from submit-queue Collect and expose runtime's image storage usage via Kubelet's /stats/summary endpoint This information is useful to users since docker images are typically not stored on the root filesystem. Kubelet will also consume this feature in the future to decide is evicting images will help with disk usage on the nodes. cc @kubernetes/sig-node
kubernetes · Apr 26, 2016 · cf38d68 · cf38d68 · k8s-teamcity-mesosphere · Apr 26, 2016
2 parents 293b0d0 + e566948
commit cf38d68
Show file tree

Hide file tree

Showing 19 changed files with 321 additions and 29 deletions.
diff --git a/pkg/kubelet/api/v1alpha1/stats/types.go b/pkg/kubelet/api/v1alpha1/stats/types.go
@@ -46,6 +46,16 @@ type NodeStats struct {
 	// Stats pertaining to total usage of filesystem resources on the rootfs used by node k8s components.
 	// NodeFs.Used is the total bytes used on the filesystem.
 	Fs *FsStats `json:"fs,omitempty"`
+	// Stats about the underlying container runtime.
+	Runtime *RuntimeStats `json:"runtime,omitempty"`
+}
+
+// Stats pertaining to the underlying container runtime.
+type RuntimeStats struct {
+	// Stats about the underlying filesystem where container images are stored.
+	// This filesystem could be the same as the primary (root) filesystem.
+	// Usage here refers to the total number of bytes occupied by images on the filesystem.
+	ImageFs *FsStats `json:"imageFs,omitempty"`
 }
 
 const (

diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go
@@ -46,6 +46,12 @@ type ImageSpec struct {
 	Image string
 }
 
+// ImageStats contains statistics about all the images currently available.
+type ImageStats struct {
+	// Total amount of storage consumed by existing images.
+	TotalStorageBytes uint64
+}
+
 // Runtime interface defines the interfaces that should be implemented
 // by a container runtime.
 // Thread safety is required from implementations of this interface.
@@ -86,6 +92,8 @@ type Runtime interface {
 	ListImages() ([]Image, error)
 	// Removes the specified image.
 	RemoveImage(image ImageSpec) error
+	// Returns Image statistics.
+	ImageStats() (*ImageStats, error)
 	// TODO(vmarmol): Unify pod and containerID args.
 	// GetContainerLogs returns logs of a specific container. By
 	// default, it returns a snapshot of the container log. Set 'follow' to true to

diff --git a/pkg/kubelet/container/testing/fake_runtime.go b/pkg/kubelet/container/testing/fake_runtime.go
@@ -353,3 +353,11 @@ func (f *FakeRuntime) GarbageCollect(gcPolicy ContainerGCPolicy) error {
 	f.CalledFunctions = append(f.CalledFunctions, "GarbageCollect")
 	return f.Err
 }
+
+func (f *FakeRuntime) ImageStats() (*ImageStats, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.CalledFunctions = append(f.CalledFunctions, "ImageStats")
+	return nil, f.Err
+}
diff --git a/pkg/kubelet/container/testing/runtime_mock.go b/pkg/kubelet/container/testing/runtime_mock.go
@@ -137,3 +137,8 @@ func (r *Mock) GarbageCollect(gcPolicy ContainerGCPolicy) error {
 	args := r.Called(gcPolicy)
 	return args.Error(0)
 }
+
+func (r *Mock) ImageStats() (*ImageStats, error) {
+	args := r.Called()
+	return args.Get(0).(*ImageStats), args.Error(1)
+}
diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go
@@ -67,6 +67,7 @@ type DockerInterface interface {
 	ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error)
 	PullImage(image string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error
 	RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error)
+	ImageHistory(id string) ([]dockertypes.ImageHistory, error)
 	Logs(string, dockertypes.ContainerLogsOptions, StreamOptions) error
 	Version() (*dockertypes.Version, error)
 	Info() (*dockertypes.Info, error)

diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go
@@ -46,15 +46,16 @@ type FakeDockerClient struct {
 	pulled               []string
 
 	// Created, Stopped and Removed all container docker ID
-	Created       []string
-	Stopped       []string
-	Removed       []string
-	RemovedImages sets.String
-	VersionInfo   dockertypes.Version
-	Information   dockertypes.Info
-	ExecInspect   *dockertypes.ContainerExecInspect
-	execCmd       []string
-	EnableSleep   bool
+	Created         []string
+	Stopped         []string
+	Removed         []string
+	RemovedImages   sets.String
+	VersionInfo     dockertypes.Version
+	Information     dockertypes.Info
+	ExecInspect     *dockertypes.ContainerExecInspect
+	execCmd         []string
+	EnableSleep     bool
+	ImageHistoryMap map[string][]dockertypes.ImageHistory
 }
 
 // We don't check docker version now, just set the docker version of fake docker client to 1.8.1.
@@ -482,6 +483,12 @@ func (f *FakeDockerClient) RemoveImage(image string, opts dockertypes.ImageRemov
 	return []dockertypes.ImageDelete{{Deleted: image}}, err
 }
 
+func (f *FakeDockerClient) InjectImages(images []dockertypes.Image) {
+	f.Lock()
+	defer f.Unlock()
+	f.Images = append(f.Images, images...)
+}
+
 func (f *FakeDockerClient) updateContainerStatus(id, status string) {
 	for i := range f.RunningContainerList {
 		if f.RunningContainerList[i].ID == id {
@@ -528,6 +535,18 @@ func (f *FakeDockerPuller) IsImagePresent(name string) (bool, error) {
 	}
 	return false, nil
 }
+func (f *FakeDockerClient) ImageHistory(id string) ([]dockertypes.ImageHistory, error) {
+	f.Lock()
+	defer f.Unlock()
+	history := f.ImageHistoryMap[id]
+	return history, nil
+}
+
+func (f *FakeDockerClient) InjectImageHistory(data map[string][]dockertypes.ImageHistory) {
+	f.Lock()
+	defer f.Unlock()
+	f.ImageHistoryMap = data
+}
 
 // dockerTimestampToString converts the timestamp to string
 func dockerTimestampToString(t time.Time) string {

diff --git a/pkg/kubelet/dockertools/images.go b/pkg/kubelet/dockertools/images.go
@@ -0,0 +1,71 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockertools
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+
+	dockertypes "github.com/docker/engine-api/types"
+	runtime "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/util/sets"
+)
+
+// imageStatsProvider exposes stats about all images currently available.
+type imageStatsProvider struct {
+	// Docker remote API client
+	c DockerInterface
+}
+
+func (isp *imageStatsProvider) ImageStats() (*runtime.ImageStats, error) {
+	images, err := isp.c.ListImages(dockertypes.ImageListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list docker images - %v", err)
+	}
+	// A map of all the image layers to its corresponding size.
+	imageMap := sets.NewString()
+	ret := &runtime.ImageStats{}
+	for _, image := range images {
+		// Get information about the various layers of each docker image.
+		history, err := isp.c.ImageHistory(image.ID)
+		if err != nil {
+			glog.V(2).Infof("failed to get history of docker image %v - %v", image, err)
+			continue
+		}
+		// Store size information of each layer.
+		for _, layer := range history {
+			// Skip empty layers.
+			if layer.Size == 0 {
+				glog.V(10).Infof("skipping image layer %v with size 0", layer)
+				continue
+			}
+			key := layer.ID
+			// Some of the layers are empty.
+			// We are hoping that these layers are unique to each image.
+			// Still keying with the CreatedBy field to be safe.
+			if key == "" || key == "<missing>" {
+				key = key + layer.CreatedBy
+			}
+			if !imageMap.Has(key) {
+				ret.TotalStorageBytes += uint64(layer.Size)
+			}
+			imageMap.Insert(key)
+		}
+	}
+	return ret, nil
+}
diff --git a/pkg/kubelet/dockertools/images_test.go b/pkg/kubelet/dockertools/images_test.go
@@ -0,0 +1,103 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockertools
+
+import (
+	"testing"
+
+	dockertypes "github.com/docker/engine-api/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestImageStatsNoImages(t *testing.T) {
+	fakeDockerClient := NewFakeDockerClientWithVersion("1.2.3", "1.2")
+	isp := &imageStatsProvider{fakeDockerClient}
+	st, err := isp.ImageStats()
+	as := assert.New(t)
+	as.NoError(err)
+	as.Equal(st.TotalStorageBytes, uint64(0))
+}
+
+func TestImageStatsWithImages(t *testing.T) {
+	fakeDockerClient := NewFakeDockerClientWithVersion("1.2.3", "1.2")
+	fakeHistoryData := map[string][]dockertypes.ImageHistory{
+		"busybox": {
+			{
+				ID:        "0123456",
+				CreatedBy: "foo",
+				Size:      100,
+			},
+			{
+				ID:        "0123457",
+				CreatedBy: "duplicate",
+				Size:      200,
+			},
+			{
+				ID:        "<missing>",
+				CreatedBy: "baz",
+				Size:      300,
+			},
+		},
+		"kubelet": {
+			{
+				ID:        "1123456",
+				CreatedBy: "foo",
+				Size:      200,
+			},
+			{
+				ID:        "<missing>",
+				CreatedBy: "1baz",
+				Size:      400,
+			},
+		},
+		"busybox-new": {
+			{
+				ID:        "01234567",
+				CreatedBy: "foo",
+				Size:      100,
+			},
+			{
+				ID:        "0123457",
+				CreatedBy: "duplicate",
+				Size:      200,
+			},
+			{
+				ID:        "<missing>",
+				CreatedBy: "baz",
+				Size:      300,
+			},
+		},
+	}
+	fakeDockerClient.InjectImageHistory(fakeHistoryData)
+	fakeDockerClient.InjectImages([]dockertypes.Image{
+		{
+			ID: "busybox",
+		},
+		{
+			ID: "kubelet",
+		},
+		{
+			ID: "busybox-new",
+		},
+	})
+	isp := &imageStatsProvider{fakeDockerClient}
+	st, err := isp.ImageStats()
+	as := assert.New(t)
+	as.NoError(err)
+	const expectedOutput uint64 = 1300
+	as.Equal(expectedOutput, st.TotalStorageBytes, "expected %d, got %d", expectedOutput, st.TotalStorageBytes)
+}
diff --git a/pkg/kubelet/dockertools/instrumented_docker.go b/pkg/kubelet/dockertools/instrumented_docker.go
@@ -199,3 +199,12 @@ func (in instrumentedDockerInterface) AttachToContainer(id string, opts dockerty
 	recordError(operation, err)
 	return err
 }
+
+func (in instrumentedDockerInterface) ImageHistory(id string) ([]dockertypes.ImageHistory, error) {
+	const operation = "image_history"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.client.ImageHistory(id)
+	recordError(operation, err)
+	return out, err
+}
diff --git a/pkg/kubelet/dockertools/kube_docker_client.go b/pkg/kubelet/dockertools/kube_docker_client.go
@@ -151,6 +151,10 @@ func (d *kubeDockerClient) InspectImage(image string) (*dockertypes.ImageInspect
 	return &resp, nil
 }
 
+func (d *kubeDockerClient) ImageHistory(id string) ([]dockertypes.ImageHistory, error) {
+	return d.client.ImageHistory(getDefaultContext(), id)
+}
+
 func (d *kubeDockerClient) ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) {
 	images, err := d.client.ImageList(getDefaultContext(), opts)
 	if err != nil {

diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go
@@ -163,6 +163,9 @@ type DockerManager struct {
 
 	// The api version cache of docker daemon.
 	versionCache *cache.VersionCache
+
+	// Provides image stats
+	*imageStatsProvider
 }
 
 // A subset of the pod.Manager interface extracted for testing purposes.
@@ -240,6 +243,7 @@ func NewDockerManager(
 		cpuCFSQuota:            cpuCFSQuota,
 		enableCustomMetrics:    enableCustomMetrics,
 		configureHairpinMode:   hairpinMode,
+		imageStatsProvider:     &imageStatsProvider{client},
 	}
 	dm.runner = lifecycle.NewHandlerRunner(httpClient, dm, dm)
 	if serializeImagePulls {

diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
@@ -342,8 +342,6 @@ func NewMainKubelet(
 		enableCustomMetrics:          enableCustomMetrics,
 		babysitDaemons:               babysitDaemons,
 	}
-	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
-	klet.resourceAnalyzer = stats.NewResourceAnalyzer(klet, volumeStatsAggPeriod)
 
 	if klet.flannelExperimentalOverlay {
 		glog.Infof("Flannel is in charge of podCIDR and overlay networking.")
@@ -440,6 +438,9 @@ func NewMainKubelet(
 		return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime)
 	}
 
+	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
+	klet.resourceAnalyzer = stats.NewResourceAnalyzer(klet, volumeStatsAggPeriod, klet.containerRuntime)
+
 	klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, util.RealClock{})
 	klet.runtimeState = newRuntimeState(maxWaitForContainerRuntime, configureCBR0)
 	klet.updatePodCIDR(podCIDR)
@@ -3579,11 +3580,11 @@ func (kl *Kubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) {
 }
 
 func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableDebuggingHandlers bool) {
-	server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableDebuggingHandlers)
+	server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableDebuggingHandlers, kl.containerRuntime)
 }
 
 func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint) {
-	server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port)
+	server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port, kl.containerRuntime)
 }
 
 // GetRuntime returns the current Runtime implementation in use by the kubelet. This func

diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go
@@ -1696,3 +1696,8 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont
 
 	return podStatus, nil
 }
+
+// FIXME: I need to be implemented.
+func (r *Runtime) ImageStats() (*kubecontainer.ImageStats, error) {
+	return &kubecontainer.ImageStats{}, nil
+}