Skip to content

Commit

Permalink
Declare out of disk when there is no free inodes.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ronnielai
ronnielai committed Jun 30, 2016
1 parent 61a9358 commit 8cc6057
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 44 deletions.
15 changes: 10 additions & 5 deletions pkg/kubelet/disk_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,11 @@ type DiskSpacePolicy struct {
}

type fsInfo struct {
Usage int64
Capacity int64
Available int64
Timestamp time.Time
Usage int64
Capacity int64
Available int64
Timestamp time.Time
InodesFree int64
}

type realDiskSpaceManager struct {
Expand Down Expand Up @@ -78,6 +79,7 @@ func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.F
fsi.Usage = int64(fs.Usage)
fsi.Capacity = int64(fs.Capacity)
fsi.Available = int64(fs.Available)
fsi.InodesFree = int64(fs.InodesFree)
dm.cachedInfo[fsType] = fsi
}
return fsi, nil
Expand All @@ -102,11 +104,14 @@ func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f
if fsInfo.Available < 0 {
return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo)
}

if fsInfo.Available < int64(threshold)*mb {
glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold)
return false, nil
}
if fsInfo.InodesFree == 0 {
glog.Infof("Running out of inodes for %q", fsType)
return false, nil
}
return true, nil
}

Expand Down
127 changes: 90 additions & 37 deletions pkg/kubelet/disk_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,15 @@ func TestSpaceAvailable(t *testing.T) {
assert.NoError(err)

mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
InodesFree: 5,
}, nil)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Usage: 9 * mb,
Capacity: 10 * mb,
InodesFree: 5,
}, nil)

ok, err := dm.IsRuntimeDiskSpaceAvailable()
Expand All @@ -81,17 +83,18 @@ func TestSpaceAvailable(t *testing.T) {
}

// TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when
// space is available.
// space and inodes are available.
func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)

// 500MB available
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9500 * mb,
Capacity: 10000 * mb,
Available: 500 * mb,
Usage: 9500 * mb,
Capacity: 10000 * mb,
Available: 500 * mb,
InodesFree: 10,
}, nil)

ok, err := dm.IsRuntimeDiskSpaceAvailable()
Expand All @@ -105,9 +108,30 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
// 1MB available
assert, policy, mockCadvisor := setUp(t)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 999 * mb,
Capacity: 1000 * mb,
Available: 1 * mb,
Usage: 999 * mb,
Capacity: 1000 * mb,
Available: 1 * mb,
InodesFree: 10,
}, nil)

dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)

ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}

// TestIsRuntimeDiskSpaceAvailableWithoutInode verifies IsRuntimeDiskSpaceAvailable results when
// inode is not available.
func TestIsRuntimeDiskSpaceAvailableWithoutInode(t *testing.T) {
// 1MB available
assert, policy, mockCadvisor := setUp(t)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 999 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
InodesFree: 0,
}, nil)

dm, err := newDiskSpaceManager(mockCadvisor, policy)
Expand All @@ -119,7 +143,7 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
}

// TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when
// space is available.
// space and inodes are available.
func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
Expand All @@ -128,9 +152,10 @@ func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {

// 999MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 1 * mb,
Capacity: 1000 * mb,
Available: 999 * mb,
Usage: 1 * mb,
Capacity: 1000 * mb,
Available: 999 * mb,
InodesFree: 10,
}, nil)

ok, err := dm.IsRootDiskSpaceAvailable()
Expand All @@ -148,9 +173,31 @@ func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) {

// 9MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 990 * mb,
Capacity: 1000 * mb,
Available: 9 * mb,
Usage: 990 * mb,
Capacity: 1000 * mb,
Available: 9 * mb,
InodesFree: 10,
}, nil)

ok, err := dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}

// TestIsRootDiskSpaceAvailableWithoutInode verifies IsRootDiskSpaceAvailable results when
// inode is not available.
func TestIsRootDiskSpaceAvailableWithoutInode(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)

// 9MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 990 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
InodesFree: 0,
}, nil)

ok, err := dm.IsRootDiskSpaceAvailable()
Expand All @@ -165,14 +212,16 @@ func TestCache(t *testing.T) {
assert.NoError(err)

mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 300 * mb,
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 300 * mb,
InodesFree: 10,
}, nil).Once()
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 500 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
Usage: 500 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
InodesFree: 5,
}, nil).Once()

// Initial calls which should be recorded in mockCadvisor
Expand Down Expand Up @@ -224,9 +273,10 @@ func Test_getFsInfo(t *testing.T) {

// Sunny day case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 10 * mb,
Capacity: 100 * mb,
Available: 90 * mb,
Usage: 10 * mb,
Capacity: 100 * mb,
Available: 90 * mb,
InodesFree: 5,
}, nil).Once()

dm := &realDiskSpaceManager{
Expand All @@ -242,9 +292,10 @@ func Test_getFsInfo(t *testing.T) {
// Threshold case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 100 * mb,
Available: 9 * mb,
Usage: 9 * mb,
Capacity: 100 * mb,
Available: 9 * mb,
InodesFree: 5,
}, nil).Once()

dm = &realDiskSpaceManager{
Expand All @@ -258,9 +309,10 @@ func Test_getFsInfo(t *testing.T) {

// Frozen case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Available: 500 * mb,
Usage: 9 * mb,
Capacity: 10 * mb,
Available: 500 * mb,
InodesFree: 5,
}, nil).Once()

dm = &realDiskSpaceManager{
Expand All @@ -275,9 +327,10 @@ func Test_getFsInfo(t *testing.T) {
// Capacity error case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 0,
Available: 500 * mb,
Usage: 9 * mb,
Capacity: 0,
Available: 500 * mb,
InodesFree: 5,
}, nil).Once()

dm = &realDiskSpaceManager{
Expand Down
4 changes: 2 additions & 2 deletions pkg/kubelet/kubelet_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2695,8 +2695,8 @@ func TestValidateContainerLogStatus(t *testing.T) {
// sufficient disk space or it is out of disk, depending on the capacity, availability and
// threshold values.
func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error {
dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb}
rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb}
dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb, InodesFree: 5}
rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb, InodesFree: 5}
mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil)
mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)

Expand Down

0 comments on commit 8cc6057

Please sign in to comment.