Merge pull request #120914 from kannon92/kubelet-disk-api-cri-update

Kubelet disk api cri update
kubernetes · Oct 24, 2023 · 413b15a · 413b15a
2 parents f652e61 + 8ae0d39
commit 413b15a
Show file tree

Hide file tree

Showing 10 changed files with 538 additions and 441 deletions.
diff --git a/pkg/kubelet/cri/remote/fake/fake_image_service.go b/pkg/kubelet/cri/remote/fake/fake_image_service.go
@@ -72,10 +72,10 @@ func (f *RemoteRuntime) RemoveImage(ctx context.Context, req *kubeapi.RemoveImag
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
 func (f *RemoteRuntime) ImageFsInfo(ctx context.Context, req *kubeapi.ImageFsInfoRequest) (*kubeapi.ImageFsInfoResponse, error) {
-	fsUsage, err := f.ImageService.ImageFsInfo(ctx)
+	resp, err := f.ImageService.ImageFsInfo(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	return &kubeapi.ImageFsInfoResponse{ImageFilesystems: fsUsage}, nil
+	return resp, nil
 }
diff --git a/pkg/kubelet/cri/remote/remote_image.go b/pkg/kubelet/cri/remote/remote_image.go
@@ -217,7 +217,7 @@ func (r *remoteImageService) RemoveImage(ctx context.Context, image *runtimeapi.
 }
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
-func (r *remoteImageService) ImageFsInfo(ctx context.Context) ([]*runtimeapi.FilesystemUsage, error) {
+func (r *remoteImageService) ImageFsInfo(ctx context.Context) (*runtimeapi.ImageFsInfoResponse, error) {
 	// Do not set timeout, because `ImageFsInfo` takes time.
 	// TODO(random-liu): Should we assume runtime should cache the result, and set timeout here?
 	ctx, cancel := context.WithCancel(ctx)
@@ -226,11 +226,11 @@ func (r *remoteImageService) ImageFsInfo(ctx context.Context) ([]*runtimeapi.Fil
 	return r.imageFsInfoV1(ctx)
 }
 
-func (r *remoteImageService) imageFsInfoV1(ctx context.Context) ([]*runtimeapi.FilesystemUsage, error) {
+func (r *remoteImageService) imageFsInfoV1(ctx context.Context) (*runtimeapi.ImageFsInfoResponse, error) {
 	resp, err := r.imageClient.ImageFsInfo(ctx, &runtimeapi.ImageFsInfoRequest{})
 	if err != nil {
 		klog.ErrorS(err, "ImageFsInfo from image service failed")
 		return nil, err
 	}
-	return resp.GetImageFilesystems(), nil
+	return resp, nil
 }
diff --git a/pkg/kubelet/kuberuntime/instrumented_services.go b/pkg/kubelet/kuberuntime/instrumented_services.go
@@ -317,7 +317,7 @@ func (in instrumentedImageManagerService) RemoveImage(ctx context.Context, image
 	return err
 }
 
-func (in instrumentedImageManagerService) ImageFsInfo(ctx context.Context) ([]*runtimeapi.FilesystemUsage, error) {
+func (in instrumentedImageManagerService) ImageFsInfo(ctx context.Context) (*runtimeapi.ImageFsInfoResponse, error) {
 	const operation = "image_fs_info"
 	defer recordOperation(operation, time.Now())
 

diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go
@@ -395,10 +395,10 @@ func (p *criStatsProvider) ImageFsStats(ctx context.Context) (*statsapi.FsStats,
 	// return the first one.
 	//
 	// TODO(yguo0905): Support returning stats of multiple image filesystems.
-	if len(resp) == 0 {
+	if len(resp.GetImageFilesystems()) == 0 {
 		return nil, fmt.Errorf("imageFs information is unavailable")
 	}
-	fs := resp[0]
+	fs := resp.GetImageFilesystems()[0]
 	s := &statsapi.FsStats{
 		Time:      metav1.NewTime(time.Unix(0, fs.Timestamp)),
 		UsedBytes: &fs.UsedBytes.Value,
@@ -430,7 +430,7 @@ func (p *criStatsProvider) ImageFsDevice(ctx context.Context) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	for _, fs := range resp {
+	for _, fs := range resp.GetImageFilesystems() {
 		fsInfo, err := p.getFsInfo(fs.GetFsId())
 		if err != nil {
 			return "", fmt.Errorf("get filesystem info: %w", err)

diff --git a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go
diff --git a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto
@@ -1583,6 +1583,11 @@ message WindowsFilesystemUsage {
 message ImageFsInfoResponse {
     // Information of image filesystem(s).
     repeated FilesystemUsage image_filesystems = 1;
+    // Information of container filesystem(s).
+    // This is an optional field, may be used for example if container and image
+    // storage are separated.
+    // Default will be to return this as empty.
+    repeated FilesystemUsage container_filesystems = 2;
 }
 
 message ContainerStatsRequest{

diff --git a/staging/src/k8s.io/cri-api/pkg/apis/services.go b/staging/src/k8s.io/cri-api/pkg/apis/services.go
@@ -131,6 +131,6 @@ type ImageManagerService interface {
 	PullImage(ctx context.Context, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error)
 	// RemoveImage removes the image.
 	RemoveImage(ctx context.Context, image *runtimeapi.ImageSpec) error
-	// ImageFsInfo returns information of the filesystem that is used to store images.
-	ImageFsInfo(ctx context.Context) ([]*runtimeapi.FilesystemUsage, error)
+	// ImageFsInfo returns information of the filesystem(s) used to store the read-only layers and the writeable layer.
+	ImageFsInfo(ctx context.Context) (*runtimeapi.ImageFsInfoResponse, error)
 }
diff --git a/staging/src/k8s.io/cri-api/pkg/apis/testing/fake_image_service.go b/staging/src/k8s.io/cri-api/pkg/apis/testing/fake_image_service.go
@@ -38,7 +38,8 @@ type FakeImageService struct {
 
 	pulledImages []*pulledImage
 
-	FakeFilesystemUsage []*runtimeapi.FilesystemUsage
+	FakeFilesystemUsage          []*runtimeapi.FilesystemUsage
+	FakeContainerFilesystemUsage []*runtimeapi.FilesystemUsage
 }
 
 // SetFakeImages sets the list of fake images for the FakeImageService.
@@ -93,6 +94,14 @@ func (r *FakeImageService) SetFakeFilesystemUsage(usage []*runtimeapi.Filesystem
 	r.FakeFilesystemUsage = usage
 }
 
+// SetFakeFilesystemUsage sets the FilesystemUsage for FakeImageService.
+func (r *FakeImageService) SetFakeContainerFilesystemUsage(usage []*runtimeapi.FilesystemUsage) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.FakeContainerFilesystemUsage = usage
+}
+
 // NewFakeImageService creates a new FakeImageService.
 func NewFakeImageService() *FakeImageService {
 	return &FakeImageService{
@@ -218,7 +227,7 @@ func (r *FakeImageService) RemoveImage(_ context.Context, image *runtimeapi.Imag
 }
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
-func (r *FakeImageService) ImageFsInfo(_ context.Context) ([]*runtimeapi.FilesystemUsage, error) {
+func (r *FakeImageService) ImageFsInfo(_ context.Context) (*runtimeapi.ImageFsInfoResponse, error) {
 	r.Lock()
 	defer r.Unlock()
 
@@ -227,7 +236,10 @@ func (r *FakeImageService) ImageFsInfo(_ context.Context) ([]*runtimeapi.Filesys
 		return nil, err
 	}
 
-	return r.FakeFilesystemUsage, nil
+	return &runtimeapi.ImageFsInfoResponse{
+		ImageFilesystems:     r.FakeFilesystemUsage,
+		ContainerFilesystems: r.FakeContainerFilesystemUsage,
+	}, nil
 }
 
 // AssertImagePulledWithAuth validates whether the image was pulled with auth and asserts if it wasn't.

diff --git a/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go b/staging/src/k8s.io/kubelet/pkg/apis/stats/v1alpha1/types.go
@@ -83,6 +83,11 @@ type RuntimeStats struct {
 	// Usage here refers to the total number of bytes occupied by images on the filesystem.
 	// +optional
 	ImageFs *FsStats `json:"imageFs,omitempty"`
+	// Stats about the underlying filesystem where container's writeable layer is stored.
+	// This filesystem could be the same as the primary (root) filesystem or the ImageFS.
+	// Usage here refers to the total number of bytes occupied by the writeable layer on the filesystem.
+	// +optional
+	ContainerFs *FsStats `json:"containerFs,omitempty"`
 }
 
 const (

diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go
@@ -317,6 +317,7 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() {
 							"Inodes":     bounded(1e4, 1e8),
 							"InodesUsed": bounded(0, 1e8),
 						}),
+						"ContainerFs": gomega.BeNil(),
 					}),
 					"Rlimit": ptrMatchAllFields(gstruct.Fields{
 						"Time":                  recent(maxStatsAge),