-
Notifications
You must be signed in to change notification settings - Fork 40k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
update GCE plugin for block support #58710
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,169 @@ | ||
/* | ||
Copyright 2018 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package gce_pd | ||
|
||
import ( | ||
"fmt" | ||
"path" | ||
"path/filepath" | ||
"strconv" | ||
|
||
"github.com/golang/glog" | ||
"k8s.io/api/core/v1" | ||
"k8s.io/apimachinery/pkg/types" | ||
"k8s.io/kubernetes/pkg/util/mount" | ||
kstrings "k8s.io/kubernetes/pkg/util/strings" | ||
"k8s.io/kubernetes/pkg/volume" | ||
"k8s.io/kubernetes/pkg/volume/util" | ||
) | ||
|
||
var _ volume.VolumePlugin = &gcePersistentDiskPlugin{} | ||
var _ volume.PersistentVolumePlugin = &gcePersistentDiskPlugin{} | ||
var _ volume.BlockVolumePlugin = &gcePersistentDiskPlugin{} | ||
var _ volume.DeletableVolumePlugin = &gcePersistentDiskPlugin{} | ||
var _ volume.ProvisionableVolumePlugin = &gcePersistentDiskPlugin{} | ||
var _ volume.ExpandableVolumePlugin = &gcePersistentDiskPlugin{} | ||
|
||
func (plugin *gcePersistentDiskPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) { | ||
pluginDir := plugin.host.GetVolumeDevicePluginDir(gcePersistentDiskPluginName) | ||
blkutil := util.NewBlockVolumePathHandler() | ||
globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID) | ||
if err != nil { | ||
return nil, err | ||
} | ||
glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err) | ||
|
||
globalMapPath := filepath.Dir(globalMapPathUUID) | ||
if len(globalMapPath) <= 1 { | ||
return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID) | ||
} | ||
|
||
return getVolumeSpecFromGlobalMapPath(globalMapPath) | ||
} | ||
|
||
func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) { | ||
// Get volume spec information from globalMapPath | ||
// globalMapPath example: | ||
// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID} | ||
// plugins/kubernetes.io/gce-pd/volumeDevices/vol-XXXXXX | ||
pdName := filepath.Base(globalMapPath) | ||
if len(pdName) <= 1 { | ||
return nil, fmt.Errorf("failed to get pd name from global path=%s", globalMapPath) | ||
} | ||
block := v1.PersistentVolumeBlock | ||
gceVolume := &v1.PersistentVolume{ | ||
Spec: v1.PersistentVolumeSpec{ | ||
PersistentVolumeSource: v1.PersistentVolumeSource{ | ||
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ | ||
PDName: pdName, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmmm partition is not being reconstructed (same with filesystem). I wonder if that could be a problem. I need to think about that a little more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When we unmap, do we need to know the actual device path? Or do we just cleanup the symlinks that only use volume name? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we need the devicePath, Unmap will clean up the sym links and the loopback devices as far as I can tell. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, In order to remove loopback device, devicePath like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So for regular scenario, the device path for partitions should be what attacher returns. For reconstruction, do we need to correctly reconstruct the partition number? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Correct.
I suppose we don't need to create full spec during reconstruction but at least, reconstructed spec must have members to create block volume mapper successfully. If block volume is created by partitioned devicePath in previous, symlink is connected to the We need this logic in @jingxu97 's next reconstruction update PR. |
||
}, | ||
}, | ||
VolumeMode: &block, | ||
}, | ||
} | ||
|
||
return volume.NewSpecFromPersistentVolume(gceVolume, true), nil | ||
} | ||
|
||
// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification. | ||
func (plugin *gcePersistentDiskPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) { | ||
// If this is called via GenerateUnmapDeviceFunc(), pod is nil. | ||
// Pass empty string as dummy uid since uid isn't used in the case. | ||
var uid types.UID | ||
if pod != nil { | ||
uid = pod.UID | ||
} | ||
|
||
return plugin.newBlockVolumeMapperInternal(spec, uid, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) | ||
} | ||
|
||
func (plugin *gcePersistentDiskPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) { | ||
volumeSource, readOnly, err := getVolumeSource(spec) | ||
if err != nil { | ||
return nil, err | ||
} | ||
pdName := volumeSource.PDName | ||
partition := "" | ||
if volumeSource.Partition != 0 { | ||
partition = strconv.Itoa(int(volumeSource.Partition)) | ||
} | ||
|
||
return &gcePersistentDiskMapper{ | ||
gcePersistentDisk: &gcePersistentDisk{ | ||
volName: spec.Name(), | ||
podUID: podUID, | ||
pdName: pdName, | ||
partition: partition, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You need to use this partition number when returning the global device path for this volume. @mtanino is it SetUp() that returns the global device path? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Or because this is attachable, is it returned in the attach method? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it is used in the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
yes, gce plugins is attachable and cloud provider supports remote attach for the plugin. |
||
manager: manager, | ||
mounter: mounter, | ||
plugin: plugin, | ||
}, | ||
readOnly: readOnly}, nil | ||
} | ||
|
||
func (plugin *gcePersistentDiskPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) { | ||
return plugin.newUnmapperInternal(volName, podUID, &GCEDiskUtil{}) | ||
} | ||
|
||
func (plugin *gcePersistentDiskPlugin) newUnmapperInternal(volName string, podUID types.UID, manager pdManager) (volume.BlockVolumeUnmapper, error) { | ||
return &gcePersistentDiskUnmapper{ | ||
gcePersistentDisk: &gcePersistentDisk{ | ||
volName: volName, | ||
podUID: podUID, | ||
pdName: volName, | ||
manager: manager, | ||
plugin: plugin, | ||
}}, nil | ||
} | ||
|
||
func (c *gcePersistentDiskUnmapper) TearDownDevice(mapPath, devicePath string) error { | ||
return nil | ||
} | ||
|
||
type gcePersistentDiskUnmapper struct { | ||
*gcePersistentDisk | ||
} | ||
|
||
var _ volume.BlockVolumeUnmapper = &gcePersistentDiskUnmapper{} | ||
|
||
type gcePersistentDiskMapper struct { | ||
*gcePersistentDisk | ||
readOnly bool | ||
} | ||
|
||
var _ volume.BlockVolumeMapper = &gcePersistentDiskMapper{} | ||
|
||
func (b *gcePersistentDiskMapper) SetUpDevice() (string, error) { | ||
return "", nil | ||
} | ||
|
||
// GetGlobalMapPath returns global map path and error | ||
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/pdName | ||
func (pd *gcePersistentDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Test case for this function? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll add one |
||
volumeSource, _, err := getVolumeSource(spec) | ||
if err != nil { | ||
return "", err | ||
} | ||
return path.Join(pd.plugin.host.GetVolumeDevicePluginDir(gcePersistentDiskPluginName), string(volumeSource.PDName)), nil | ||
} | ||
|
||
// GetPodDeviceMapPath returns pod device map path and volume name | ||
// path: pods/{podUid}/volumeDevices/kubernetes.io~aws | ||
func (pd *gcePersistentDisk) GetPodDeviceMapPath() (string, string) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Test case? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll add one |
||
name := gcePersistentDiskPluginName | ||
return pd.plugin.host.GetPodVolumeDeviceDir(pd.podUID, kstrings.EscapeQualifiedNameForDisk(name)), pd.volName | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,145 @@ | ||
/* | ||
Copyright 2018 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package gce_pd | ||
|
||
import ( | ||
"os" | ||
"path" | ||
"testing" | ||
|
||
"k8s.io/api/core/v1" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/types" | ||
utiltesting "k8s.io/client-go/util/testing" | ||
"k8s.io/kubernetes/pkg/volume" | ||
volumetest "k8s.io/kubernetes/pkg/volume/testing" | ||
) | ||
|
||
const ( | ||
testPdName = "pdVol1" | ||
testPVName = "pv1" | ||
testGlobalPath = "plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1" | ||
testPodPath = "pods/poduid/volumeDevices/kubernetes.io~gce-pd" | ||
) | ||
|
||
func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) { | ||
// make our test path for fake GlobalMapPath | ||
// /tmp symbolized our pluginDir | ||
// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/gce-pd/volumeDevices/pdVol1 | ||
tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest") | ||
if err != nil { | ||
t.Fatalf("can't make a temp dir: %v", err) | ||
} | ||
//deferred clean up | ||
defer os.RemoveAll(tmpVDir) | ||
|
||
expectedGlobalPath := path.Join(tmpVDir, testGlobalPath) | ||
|
||
//Bad Path | ||
badspec, err := getVolumeSpecFromGlobalMapPath("") | ||
if badspec != nil || err == nil { | ||
t.Errorf("Expected not to get spec from GlobalMapPath but did") | ||
} | ||
|
||
// Good Path | ||
spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath) | ||
if spec == nil || err != nil { | ||
t.Fatalf("Failed to get spec from GlobalMapPath: %v", err) | ||
} | ||
if spec.PersistentVolume.Spec.GCEPersistentDisk.PDName != testPdName { | ||
t.Errorf("Invalid pdName from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName) | ||
} | ||
block := v1.PersistentVolumeBlock | ||
specMode := spec.PersistentVolume.Spec.VolumeMode | ||
if &specMode == nil { | ||
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block) | ||
} | ||
if *specMode != block { | ||
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block) | ||
} | ||
} | ||
|
||
func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec { | ||
pv := &v1.PersistentVolume{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: testPVName, | ||
}, | ||
Spec: v1.PersistentVolumeSpec{ | ||
PersistentVolumeSource: v1.PersistentVolumeSource{ | ||
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ | ||
PDName: testPdName, | ||
}, | ||
}, | ||
}, | ||
} | ||
|
||
if isBlock { | ||
blockMode := v1.PersistentVolumeBlock | ||
pv.Spec.VolumeMode = &blockMode | ||
} | ||
return volume.NewSpecFromPersistentVolume(pv, readOnly) | ||
} | ||
|
||
func TestGetPodAndPluginMapPaths(t *testing.T) { | ||
tmpVDir, err := utiltesting.MkTmpdir("gceBlockTest") | ||
if err != nil { | ||
t.Fatalf("can't make a temp dir: %v", err) | ||
} | ||
//deferred clean up | ||
defer os.RemoveAll(tmpVDir) | ||
|
||
expectedGlobalPath := path.Join(tmpVDir, testGlobalPath) | ||
expectedPodPath := path.Join(tmpVDir, testPodPath) | ||
|
||
spec := getTestVolume(false, tmpVDir, true /*isBlock*/) | ||
plugMgr := volume.VolumePluginMgr{} | ||
plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil)) | ||
plug, err := plugMgr.FindMapperPluginByName(gcePersistentDiskPluginName) | ||
if err != nil { | ||
os.RemoveAll(tmpVDir) | ||
t.Fatalf("Can't find the plugin by name: %q", gcePersistentDiskPluginName) | ||
} | ||
if plug.GetPluginName() != gcePersistentDiskPluginName { | ||
t.Fatalf("Wrong name: %s", plug.GetPluginName()) | ||
} | ||
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} | ||
mapper, err := plug.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{}) | ||
if err != nil { | ||
t.Fatalf("Failed to make a new Mounter: %v", err) | ||
} | ||
if mapper == nil { | ||
t.Fatalf("Got a nil Mounter") | ||
} | ||
|
||
//GetGlobalMapPath | ||
gMapPath, err := mapper.GetGlobalMapPath(spec) | ||
if err != nil || len(gMapPath) == 0 { | ||
t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.GCEPersistentDisk.PDName) | ||
} | ||
if gMapPath != expectedGlobalPath { | ||
t.Errorf("Failed to get GlobalMapPath: %s %s", gMapPath, expectedGlobalPath) | ||
} | ||
|
||
//GetPodDeviceMapPath | ||
gDevicePath, gVolName := mapper.GetPodDeviceMapPath() | ||
if gDevicePath != expectedPodPath { | ||
t.Errorf("Got unexpected pod path: %s, expected %s", gDevicePath, expectedPodPath) | ||
} | ||
if gVolName != testPVName { | ||
t.Errorf("Got unexpected volNamne: %s, expected %s", gVolName, testPVName) | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Again, why 1?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same,
Base
returns "." for an empty path so would indicate an error condition