Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Csi translation portworx #103447

Merged
merged 1 commit into from
Nov 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions cmd/kube-controller-manager/app/plugins.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import (
"k8s.io/kubernetes/pkg/volume/iscsi"
"k8s.io/kubernetes/pkg/volume/local"
"k8s.io/kubernetes/pkg/volume/nfs"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/quobyte"
"k8s.io/kubernetes/pkg/volume/storageos"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
Expand All @@ -62,7 +61,6 @@ func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) {
if err != nil {
return allPlugins, err
}
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
Expand All @@ -85,7 +83,6 @@ func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfigurat
if err != nil {
return allPlugins, err
}
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
Expand Down Expand Up @@ -137,7 +134,6 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste
}

allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)

Expand Down
2 changes: 2 additions & 0 deletions cmd/kube-controller-manager/app/plugins_providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"k8s.io/kubernetes/pkg/volume/cinder"
"k8s.io/kubernetes/pkg/volume/csimigration"
"k8s.io/kubernetes/pkg/volume/gcepd"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/rbd"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
)
Expand Down Expand Up @@ -68,6 +69,7 @@ func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, fea
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins}
pluginMigrationStatus[plugins.PortworxVolumePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationPortworx, pluginUnregisterFeature: features.InTreePluginPortworxUnregister, pluginProbeFunction: portworx.ProbeVolumePlugins}
pluginMigrationStatus[plugins.RBDVolumePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationRBD, pluginUnregisterFeature: features.InTreePluginRBDUnregister, pluginProbeFunction: rbd.ProbeVolumePlugins}
var err error
for pluginName, pluginInfo := range pluginMigrationStatus {
Expand Down
2 changes: 0 additions & 2 deletions cmd/kubelet/app/plugins.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ import (
"k8s.io/kubernetes/pkg/volume/iscsi"
"k8s.io/kubernetes/pkg/volume/local"
"k8s.io/kubernetes/pkg/volume/nfs"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/projected"
"k8s.io/kubernetes/pkg/volume/quobyte"
"k8s.io/kubernetes/pkg/volume/secret"
Expand Down Expand Up @@ -76,7 +75,6 @@ func ProbeVolumePlugins(featureGate featuregate.FeatureGate) ([]volume.VolumePlu
allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
Expand Down
2 changes: 2 additions & 0 deletions cmd/kubelet/app/plugins_providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"k8s.io/kubernetes/pkg/volume/cinder"
"k8s.io/kubernetes/pkg/volume/csimigration"
"k8s.io/kubernetes/pkg/volume/gcepd"
"k8s.io/kubernetes/pkg/volume/portworx"
"k8s.io/kubernetes/pkg/volume/rbd"
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
)
Expand Down Expand Up @@ -75,6 +76,7 @@ func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate f
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, pluginProbeFunction: azure_file.ProbeVolumePlugins}
pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins}
pluginMigrationStatus[plugins.PortworxVolumePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationPortworx, pluginUnregisterFeature: features.InTreePluginPortworxUnregister, pluginProbeFunction: portworx.ProbeVolumePlugins}
pluginMigrationStatus[plugins.RBDVolumePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationRBD, pluginUnregisterFeature: features.InTreePluginRBDUnregister, pluginProbeFunction: rbd.ProbeVolumePlugins}
var err error
for pluginName, pluginInfo := range pluginMigrationStatus {
Expand Down
18 changes: 16 additions & 2 deletions pkg/features/kube_features.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,18 @@ const (
// Disables the OpenStack Cinder in-tree driver.
InTreePluginOpenStackUnregister featuregate.Feature = "InTreePluginOpenStackUnregister"

// owner: @trierra
// alpha: v1.23
//
// Enables the Portworx in-tree driver to Portworx migration feature.
CSIMigrationPortworx featuregate.Feature = "CSIMigrationPortworx"

// owner: @trierra
// alpha: v1.23
//
// Disables the Portworx in-tree driver.
InTreePluginPortworxUnregister featuregate.Feature = "InTreePluginPortworxUnregister"

// owner: @humblec
// alpha: v1.23
//
Expand Down Expand Up @@ -858,6 +870,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
CSIMigrationRBD: {Default: false, PreRelease: featuregate.Alpha}, // Off by default (requires RBD CSI driver)
InTreePluginRBDUnregister: {Default: false, PreRelease: featuregate.Alpha},
ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
CSIMigrationPortworx: {Default: false, PreRelease: featuregate.Alpha}, // Off by default (requires Portworx CSI driver)
InTreePluginPortworxUnregister: {Default: false, PreRelease: featuregate.Alpha},
CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta},
CSIStorageCapacity: {Default: true, PreRelease: featuregate.Beta},
CSIServiceAccountToken: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23
Expand All @@ -880,7 +894,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
EndpointSliceProxying: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
EndpointSliceTerminatingCondition: {Default: true, PreRelease: featuregate.Beta},
ProxyTerminatingEndpoints: {Default: false, PreRelease: featuregate.Alpha},
EndpointSliceNodeName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, //remove in 1.25
EndpointSliceNodeName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
WindowsEndpointSliceProxying: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
PodDisruptionBudget: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
DaemonSetUpdateSurge: {Default: true, PreRelease: featuregate.Beta}, // on by default in 1.22
Expand All @@ -889,7 +903,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
DownwardAPIHugePages: {Default: true, PreRelease: featuregate.Beta}, // on by default in 1.22
AnyVolumeDataSource: {Default: false, PreRelease: featuregate.Alpha},
DefaultPodTopologySpread: {Default: true, PreRelease: featuregate.Beta},
SetHostnameAsFQDN: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, //remove in 1.24
SetHostnameAsFQDN: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.24
WinOverlay: {Default: true, PreRelease: featuregate.Beta},
WinDSR: {Default: false, PreRelease: featuregate.Alpha},
DisableAcceleratorUsageMetrics: {Default: true, PreRelease: featuregate.Beta},
Expand Down
4 changes: 4 additions & 0 deletions pkg/scheduler/framework/plugins/nodevolumelimits/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool {
if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWS) {
return false
}
case csilibplugins.PortworxVolumePluginName:
if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Scheduler changes LGTM.

Will approve once sig-storage approves the feature gate's addition.

return false
}
case csilibplugins.GCEPDInTreePluginName:
if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCE) {
return false
Expand Down
4 changes: 3 additions & 1 deletion pkg/scheduler/framework/plugins/volumebinding/binder.go
Original file line number Diff line number Diff line change
Expand Up @@ -1014,7 +1014,7 @@ func (a byPVCSize) Less(i, j int) bool {
return iSize.Cmp(jSize) == -1
}

// isCSIMigrationOnForPlugin checks if CSI migrartion is enabled for a given plugin.
// isCSIMigrationOnForPlugin checks if CSI migration is enabled for a given plugin.
func isCSIMigrationOnForPlugin(pluginName string) bool {
switch pluginName {
case csiplugins.AWSEBSInTreePluginName:
Expand All @@ -1025,6 +1025,8 @@ func isCSIMigrationOnForPlugin(pluginName string) bool {
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDisk)
case csiplugins.CinderInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStack)
case csiplugins.PortworxVolumePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx)
case csiplugins.RBDVolumePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationRBD)
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/volume/csi/csi_plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,9 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
csitranslationplugins.VSphereInTreePluginName: func() bool {
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphere)
},
csitranslationplugins.PortworxVolumePluginName: func() bool {
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx)
},
csitranslationplugins.RBDVolumePluginName: func() bool {
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationRBD)
},
Expand Down
4 changes: 4 additions & 0 deletions pkg/volume/csimigration/plugin_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool {
return pm.featureGate.Enabled(features.InTreePluginOpenStackUnregister)
case csilibplugins.VSphereInTreePluginName:
return pm.featureGate.Enabled(features.InTreePluginvSphereUnregister)
case csilibplugins.PortworxVolumePluginName:
return pm.featureGate.Enabled(features.InTreePluginPortworxUnregister)
case csilibplugins.RBDVolumePluginName:
return pm.featureGate.Enabled(features.InTreePluginRBDUnregister)
default:
Expand Down Expand Up @@ -100,6 +102,8 @@ func (pm PluginManager) IsMigrationEnabledForPlugin(pluginName string) bool {
return pm.featureGate.Enabled(features.CSIMigrationOpenStack)
case csilibplugins.VSphereInTreePluginName:
return pm.featureGate.Enabled(features.CSIMigrationvSphere)
case csilibplugins.PortworxVolumePluginName:
return pm.featureGate.Enabled(features.CSIMigrationPortworx)
case csilibplugins.RBDVolumePluginName:
return pm.featureGate.Enabled(features.CSIMigrationRBD)
default:
Expand Down
12 changes: 9 additions & 3 deletions pkg/volume/portworx/portworx.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@ package portworx

import (
"fmt"
"os"

volumeclient "github.com/libopenstorage/openstorage/api/client/volume"
"k8s.io/klog/v2"
"k8s.io/mount-utils"
utilstrings "k8s.io/utils/strings"
"os"

volumeclient "github.com/libopenstorage/openstorage/api/client/volume"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
)
Expand Down Expand Up @@ -62,6 +63,11 @@ func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(portworxVolumePluginName), volName)
}

func (plugin *portworxVolumePlugin) IsMigratedToCSI() bool {
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) &&
utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx)
}

func (plugin *portworxVolumePlugin) Init(host volume.VolumeHost) error {
client, err := volumeclient.NewDriverClient(
fmt.Sprintf("http://%s:%d", host.GetHostName(), osdMgmtDefaultPort),
Expand Down
145 changes: 145 additions & 0 deletions staging/src/k8s.io/csi-translation-lib/plugins/portworx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package plugins

import (
"fmt"

"k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
PortworxVolumePluginName = "kubernetes.io/portworx-volume"
PortworxDriverName = "pxd.portworx.com"
)

var _ InTreePlugin = &portworxCSITranslator{}

type portworxCSITranslator struct{}

func NewPortworxCSITranslator() InTreePlugin {
return &portworxCSITranslator{}
}

// TranslateInTreeStorageClassToCSI takes in-tree storage class used by in-tree plugin
// and translates them to a storageclass consumable by CSI plugin
func (p portworxCSITranslator) TranslateInTreeStorageClassToCSI(sc *storagev1.StorageClass) (*storagev1.StorageClass, error) {
if sc == nil {
return nil, fmt.Errorf("sc is nil")
}
sc.Provisioner = PortworxDriverName
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Portworx The actual flag is called fs and is the same across in-tree storage class and csi storage class

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok CSI has defined a first class fstype key that allows it to get passed down as a CSI parameter. You may want to consider updating the driver to support that, but this is fine for now since the driver doesn't support it.

return sc, nil
}

// TranslateInTreeInlineVolumeToCSI takes a inline volume and will translate
// the in-tree inline volume source to a CSIPersistentVolumeSource
func (p portworxCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume, podNamespace string) (*v1.PersistentVolume, error) {
if volume == nil || volume.PortworxVolume == nil {
return nil, fmt.Errorf("volume is nil or PortworxVolume not defined on volume")
}

var am v1.PersistentVolumeAccessMode
if volume.PortworxVolume.ReadOnly {
am = v1.ReadOnlyMany
} else {
am = v1.ReadWriteOnce
}

pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", PortworxDriverName, volume.PortworxVolume.VolumeID),
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{
Driver: PortworxDriverName,
VolumeHandle: volume.PortworxVolume.VolumeID,
FSType: volume.PortworxVolume.FSType,
VolumeAttributes: make(map[string]string),
},
},
AccessModes: []v1.PersistentVolumeAccessMode{am},
},
}
return pv, nil
}

// TranslateInTreePVToCSI takes a Portworx persistent volume and will translate
// the in-tree pv source to a CSI Source
func (p portworxCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
if pv == nil || pv.Spec.PortworxVolume == nil {
return nil, fmt.Errorf("pv is nil or PortworxVolume not defined on pv")
}
csiSource := &v1.CSIPersistentVolumeSource{
Driver: PortworxDriverName,
VolumeHandle: pv.Spec.PortworxVolume.VolumeID,
FSType: pv.Spec.PortworxVolume.FSType,
VolumeAttributes: make(map[string]string), // copy access mode
}
pv.Spec.PortworxVolume = nil
pv.Spec.CSI = csiSource

return pv, nil
}

// TranslateCSIPVToInTree takes a PV with a CSI PersistentVolume Source and will translate
// it to a in-tree Persistent Volume Source for the in-tree volume
func (p portworxCSITranslator) TranslateCSIPVToInTree(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
if pv == nil || pv.Spec.CSI == nil {
return nil, fmt.Errorf("pv is nil or CSI source not defined on pv")
}
csiSource := pv.Spec.CSI

portworxSource := &v1.PortworxVolumeSource{
VolumeID: csiSource.VolumeHandle,
FSType: csiSource.FSType,
ReadOnly: csiSource.ReadOnly,
}
pv.Spec.CSI = nil
pv.Spec.PortworxVolume = portworxSource

return pv, nil
}

// CanSupport tests whether the plugin supports a given persistent volume
// specification from the API.
func (p portworxCSITranslator) CanSupport(pv *v1.PersistentVolume) bool {
return pv != nil && pv.Spec.PortworxVolume != nil
}

// CanSupportInline tests whether the plugin supports a given inline volume
// specification from the API.
func (p portworxCSITranslator) CanSupportInline(volume *v1.Volume) bool {
return volume != nil && volume.PortworxVolume != nil
}

// GetInTreePluginName returns the in-tree plugin name this migrates
func (p portworxCSITranslator) GetInTreePluginName() string {
return PortworxVolumePluginName
}

// GetCSIPluginName returns the name of the CSI plugin that supersedes the in-tree plugin
func (p portworxCSITranslator) GetCSIPluginName() string {
return PortworxDriverName
}

// RepairVolumeHandle generates a correct volume handle based on node ID information.
func (p portworxCSITranslator) RepairVolumeHandle(volumeHandle, nodeID string) (string, error) {
return volumeHandle, nil
}
Loading