Merge pull request rook#13420 from iPraveenParihar/api/csi-config-map

csi: use ceph-csi exposed csi config map API
koor-tech · Jan 16, 2024 · 591ddf2 · 591ddf2
2 parents 2ba90e9 + dfd3d62
commit 591ddf2
Show file tree

Hide file tree

Showing 10 changed files with 258 additions and 177 deletions.
diff --git a/go.mod b/go.mod
@@ -51,6 +51,7 @@ require (
 	github.com/ansel1/merry/v2 v2.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
+	github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containernetworking/cni v1.1.2 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect

diff --git a/go.sum b/go.sum
@@ -127,6 +127,8 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r
 github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
 github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83 h1:xWhLO5MR+diAsZoOcPe0zVe+JcJrqMaVbScShye6pXw=
+github.com/ceph/ceph-csi/api v0.0.0-20231227104434-06f9a98b7a83/go.mod h1:ZSvtS90FCB/becFi/rjy85sSw1igchaWZfUigxN9FxY=
 github.com/ceph/go-ceph v0.25.0 h1:sorUSVkm0F7tYKrv8afd1eJ8UXwxkYIio9o0xLTX59E=
 github.com/ceph/go-ceph v0.25.0/go.mod h1:HoEJSH32bMcGzmsqJNmSVeYrrcetSxnMfVnGIXBE59Q=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod
@@ -7,13 +7,22 @@ require (
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
 	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1
 	github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1
-	github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
 )
 
+require (
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/onsi/gomega v1.30.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/client-go v0.29.0 // indirect
+	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)
+
 require (
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/containernetworking/cni v1.1.2 // indirect
@@ -30,7 +39,6 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -52,7 +60,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/onsi/gomega v1.30.0 // indirect
+	github.com/openshift/api v0.0.0-20231204192004-bfea29e5e6c4
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -63,19 +71,14 @@ require (
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.16.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/client-go v0.29.0 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a // indirect
-	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 exclude (

diff --git a/pkg/operator/ceph/cluster/cluster_external.go b/pkg/operator/ceph/cluster/cluster_external.go
@@ -20,6 +20,7 @@ package cluster
 import (
 	"context"
 
+	cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
@@ -129,7 +130,13 @@ func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error
 
 	// Save CSI configmap
 	monEndpoints := csi.MonEndpoints(cluster.ClusterInfo.Monitors, cluster.Spec.RequireMsgr2())
-	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: cluster.ClusterInfo.Namespace, Monitors: monEndpoints})
+	csiConfigEntry := &csi.CSIClusterConfigEntry{
+		Namespace: cluster.ClusterInfo.Namespace,
+		ClusterInfo: cephcsi.ClusterInfo{
+			Monitors: monEndpoints,
+		},
+	}
+	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.ClusterInfo, csiConfigEntry)
 	if err != nil {
 		return errors.Wrap(err, "failed to update csi cluster config")
 	}

diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go
@@ -49,6 +49,8 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
 )
 
 const (
@@ -1104,7 +1106,13 @@ func (c *Cluster) saveMonConfig() error {
 	}
 
 	monEndpoints := csi.MonEndpoints(c.ClusterInfo.Monitors, c.spec.RequireMsgr2())
-	if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, &csi.CsiClusterConfigEntry{Namespace: c.ClusterInfo.Namespace, Monitors: monEndpoints}); err != nil {
+	csiConfigEntry := &csi.CSIClusterConfigEntry{
+		Namespace: c.ClusterInfo.Namespace,
+		ClusterInfo: cephcsi.ClusterInfo{
+			Monitors: monEndpoints,
+		},
+	}
+	if err := csi.SaveClusterConfig(c.context.Clientset, c.Namespace, c.ClusterInfo, csiConfigEntry); err != nil {
 		return errors.Wrap(err, "failed to update csi cluster config")
 	}
 

diff --git a/pkg/operator/ceph/csi/cluster_config.go b/pkg/operator/ceph/csi/cluster_config.go
@@ -33,47 +33,27 @@ import (
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+
+	cephcsi "github.com/ceph/ceph-csi/api/deploy/kubernetes"
 )
 
 var (
 	logger      = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-csi")
 	configMutex sync.Mutex
 )
 
-type CsiClusterConfigEntry struct {
-	ClusterID      string                   `json:"clusterID"`
-	Monitors       []string                 `json:"monitors"`
-	Namespace      string                   `json:"namespace"`
-	CephFS         *CsiCephFSSpec           `json:"cephFS,omitempty"`
-	NFS            *CsiNFSSpec              `json:"nfs,omitempty"`
-	RBD            *CsiRBDSpec              `json:"rbd,omitempty"`
-	RadosNamespace string                   `json:"radosNamespace,omitempty"`
-	ReadAffinity   *cephv1.ReadAffinitySpec `json:"readAffinity,omitempty"`
-}
-
-type CsiCephFSSpec struct {
-	NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
-	SubvolumeGroup       string `json:"subvolumeGroup,omitempty"`
-	KernelMountOptions   string `json:"kernelMountOptions,omitempty"`
-	FuseMountOptions     string `json:"fuseMountOptions,omitempty"`
-}
-
-type CsiNFSSpec struct {
-	NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
+type CSIClusterConfigEntry struct {
+	cephcsi.ClusterInfo
+	Namespace string `json:"namespace"`
 }
 
-type CsiRBDSpec struct {
-	NetNamespaceFilePath string `json:"netNamespaceFilePath,omitempty"`
-	RadosNamespace       string `json:"radosNamespace,omitempty"`
-}
-
-type csiClusterConfig []CsiClusterConfigEntry
+type csiClusterConfig []CSIClusterConfigEntry
 
 // FormatCsiClusterConfig returns a json-formatted string containing
 // the cluster-to-mon mapping required to configure ceph csi.
 func FormatCsiClusterConfig(
-	clusterKey string, mons map[string]*cephclient.MonInfo) (string, error) {
-
+	clusterKey string, mons map[string]*cephclient.MonInfo,
+) (string, error) {
 	cc := make(csiClusterConfig, 1)
 	cc[0].ClusterID = clusterKey
 	cc[0].Monitors = []string{}
@@ -125,10 +105,10 @@ func MonEndpoints(mons map[string]*cephclient.MonInfo, requireMsgr2 bool) []stri
 
 // updateCsiClusterConfig returns a json-formatted string containing
 // the cluster-to-mon mapping required to configure ceph csi.
-func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CsiClusterConfigEntry) (string, error) {
+func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *CSIClusterConfigEntry) (string, error) {
 	var (
 		cc     csiClusterConfig
-		centry CsiClusterConfigEntry
+		centry CSIClusterConfigEntry
 		found  bool
 	)
 
@@ -160,20 +140,15 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
 				break
 			}
 			centry.Monitors = newCsiClusterConfigEntry.Monitors
-			if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
+			if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
 				centry.CephFS = newCsiClusterConfigEntry.CephFS
 			}
-			if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
+			if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
 				centry.NFS = newCsiClusterConfigEntry.NFS
 			}
-			if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "") {
+			if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.RBD.NetNamespaceFilePath != "" {
 				centry.RBD = newCsiClusterConfigEntry.RBD
 			}
-			// This maintains backward compatibility for existing clusters, from now on the
-			// preferred way is to use RBD.RadosNamespace
-			if newCsiClusterConfigEntry.RadosNamespace != "" {
-				centry.RadosNamespace = newCsiClusterConfigEntry.RadosNamespace
-			}
 			found = true
 			cc[i] = centry
 			break
@@ -186,14 +161,14 @@ func updateCsiClusterConfig(curr, clusterKey string, newCsiClusterConfigEntry *C
 			centry.ClusterID = clusterKey
 			centry.Namespace = newCsiClusterConfigEntry.Namespace
 			centry.Monitors = newCsiClusterConfigEntry.Monitors
-			if newCsiClusterConfigEntry.RBD != nil && (newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
+			if newCsiClusterConfigEntry.RBD.RadosNamespace != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
 				centry.RBD = newCsiClusterConfigEntry.RBD
 			}
 			// Add a condition not to fill with empty values
-			if newCsiClusterConfigEntry.CephFS != nil && (newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "") {
+			if newCsiClusterConfigEntry.CephFS.SubvolumeGroup != "" || newCsiClusterConfigEntry.CephFS.NetNamespaceFilePath != "" {
 				centry.CephFS = newCsiClusterConfigEntry.CephFS
 			}
-			if newCsiClusterConfigEntry.NFS != nil && newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
+			if newCsiClusterConfigEntry.NFS.NetNamespaceFilePath != "" {
 				centry.NFS = newCsiClusterConfigEntry.NFS
 			}
 			cc = append(cc, centry)
@@ -239,7 +214,7 @@ func CreateCsiConfigMap(ctx context.Context, namespace string, clientset kuberne
 // value that is provided to ceph-csi uses in the storage class.
 // The locker l is typically a mutex and is used to prevent the config
 // map from being updated for multiple clusters simultaneously.
-func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CsiClusterConfigEntry) error {
+func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string, clusterInfo *cephclient.ClusterInfo, newCsiClusterConfigEntry *CSIClusterConfigEntry) error {
 	// csi is deployed into the same namespace as the operator
 	csiNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
 	if csiNamespace == "" {
@@ -286,7 +261,8 @@ func SaveClusterConfig(clientset kubernetes.Interface, clusterNamespace string,
 // updateCSIDriverOptions updates the CSI driver options, including read affinity, kernel mount options
 // and fuse mount options, for all entries belonging to the same cluster.
 func updateCSIDriverOptions(curr, clusterKey string,
-	csiDriverOptions *cephv1.CSIDriverSpec) (string, error) {
+	csiDriverOptions *cephv1.CSIDriverSpec,
+) (string, error) {
 	cc, err := parseCsiClusterConfig(curr)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to parse current csi cluster config")
@@ -295,15 +271,13 @@ func updateCSIDriverOptions(curr, clusterKey string,
 	for i := range cc {
 		// If the clusterID belongs to the same cluster, update the entry.
 		if clusterKey == cc[i].Namespace {
-			cc[i].ReadAffinity = &csiDriverOptions.ReadAffinity
-			if cc[i].CephFS == nil {
-				cc[i].CephFS = &CsiCephFSSpec{
-					KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions,
-					FuseMountOptions:   csiDriverOptions.CephFS.FuseMountOptions,
-				}
-			} else {
-				cc[i].CephFS.KernelMountOptions = csiDriverOptions.CephFS.KernelMountOptions
-				cc[i].CephFS.FuseMountOptions = csiDriverOptions.CephFS.FuseMountOptions
+			cc[i].ReadAffinity = cephcsi.ReadAffinity{
+				Enabled:             csiDriverOptions.ReadAffinity.Enabled,
+				CrushLocationLabels: csiDriverOptions.ReadAffinity.CrushLocationLabels,
+			}
+			cc[i].CephFS = cephcsi.CephFS{
+				KernelMountOptions: csiDriverOptions.CephFS.KernelMountOptions,
+				FuseMountOptions:   csiDriverOptions.CephFS.FuseMountOptions,
 			}
 		}
 	}