feat(system-backup/restore): system-rollout controller

Ref: 1455 Signed-off-by: Chin-Ya Huang <chin-ya.huang@suse.com>
c3y1huang · Dec 8, 2022 · 08b838b · 08b838b
1 parent cf08932
commit 08b838b
Show file tree

Hide file tree

Showing 19 changed files with 2,160 additions and 32 deletions.
diff --git a/constant/events.go b/constant/events.go
@@ -25,6 +25,7 @@ const (
 	EventReasonTimeoutSnapshotPurge        = "TimeoutSnapshotPurge"
 	EventReasonFailedSnapshotPurge         = "FailedSnapshotPurge"
 
+	EventReasonRestored      = "Restored"
 	EventReasonFailedRestore = "FailedRestore"
 
 	EventReasonFailedExpansion    = "FailedExpansion"
@@ -45,8 +46,10 @@ const (
 	EventReasonAutoSalvaged       = "AutoSalvaged"
 
 	EventReasonFetching = "Fetching"
-	EventReasonSyncing  = "Syncing"
-	EventReasonSynced   = "Synced"
+	EventReasonFetched  = "Fetched"
+
+	EventReasonSyncing = "Syncing"
+	EventReasonSynced  = "Synced"
 
 	EventReasonFailedSnapshotDataIntegrityCheck = "FailedSnapshotDataIntegrityCheck"
 

diff --git a/controller/backup_controller.go b/controller/backup_controller.go
@@ -272,7 +272,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) {
 
 		if backupTarget.Spec.BackupTargetURL != "" &&
 			backupVolume != nil && backupVolume.DeletionTimestamp == nil {
-			backupTargetClient, err := getBackupTargetClient(bc.ds, backupTarget)
+			backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget)
 			if err != nil {
 				log.WithError(err).Error("Error init backup target clients")
 				return nil // Ignore error to prevent enqueue
@@ -374,7 +374,7 @@ func (bc *BackupController) reconcile(backupName string) (err error) {
 	}
 
 	// The backup creation is complete, then the source of truth becomes the remote backup target
-	backupTargetClient, err := getBackupTargetClient(bc.ds, backupTarget)
+	backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(bc.ds, backupTarget)
 	if err != nil {
 		log.WithError(err).Error("Error init backup target clients")
 		return nil // Ignore error to prevent enqueue

diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go
@@ -222,7 +222,7 @@ func getBackupTarget(controllerID string, backupTarget *longhorn.BackupTarget, d
 		return nil, nil, err
 	}
 
-	backupTargetClient, err = getBackupTargetClient(ds, backupTarget)
+	backupTargetClient, err = newBackupTargetClientFromDefaultEngineImage(ds, backupTarget)
 	if err != nil {
 		engineClientProxy.Close()
 		return nil, nil, err
@@ -231,11 +231,10 @@ func getBackupTarget(controllerID string, backupTarget *longhorn.BackupTarget, d
 	return engineClientProxy, backupTargetClient, nil
 }
 
-func getBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget) (*engineapi.BackupTargetClient, error) {
-	defaultEngineImage, err := ds.GetSettingValueExisted(types.SettingNameDefaultEngineImage)
-	if err != nil {
-		return nil, err
-	}
+func newBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget, engineImage string) (backupTargetClient *engineapi.BackupTargetClient, err error) {
+	defer func() {
+		err = errors.Wrapf(err, "failed to get %v backup target client on %v", backupTarget.Name, engineImage)
+	}()
 
 	backupType, err := util.CheckBackupType(backupTarget.Spec.BackupTargetURL)
 	if err != nil {
@@ -252,21 +251,16 @@ func getBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.Backu
 			return nil, err
 		}
 	}
-	return engineapi.NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential), nil
+	return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential), nil
 }
 
-func getBackupTargetClientDefault(ds *datastore.DataStore) (*engineapi.BackupTargetClient, error) {
-	backupTarget, err := ds.GetDefaultBackupTargetRO()
+func newBackupTargetClientFromDefaultEngineImage(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget) (*engineapi.BackupTargetClient, error) {
+	defaultEngineImage, err := ds.GetSettingValueExisted(types.SettingNameDefaultEngineImage)
 	if err != nil {
 		return nil, err
 	}
 
-	backupTargetClient, err := getBackupTargetClient(ds, backupTarget)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get default backup target client")
-	}
-
-	return backupTargetClient, nil
+	return newBackupTargetClient(ds, backupTarget, defaultEngineImage)
 }
 
 func (btc *BackupTargetController) reconcile(name string) (err error) {

diff --git a/controller/engine_controller.go b/controller/engine_controller.go
@@ -1238,7 +1238,7 @@ func (m *EngineMonitor) restoreBackup(engine *longhorn.Engine, rsMap map[string]
 		return fmt.Errorf("cannot found the %s backup target", types.DefaultBackupTargetName)
 	}
 
-	backupTargetClient, err := getBackupTargetClient(m.ds, backupTarget)
+	backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(m.ds, backupTarget)
 	if err != nil {
 		return errors.Wrapf(err, "cannot get backup target config for backup restoration of engine %v", engine.Name)
 	}

diff --git a/controller/share_manager_controller.go b/controller/share_manager_controller.go
@@ -300,7 +300,8 @@ func (c *ShareManagerController) syncShareManager(key string) (err error) {
 			return err
 		}
 
-		if err := c.ds.DeleteConfigMap(c.namespace, types.GetConfigMapNameFromShareManagerName(sm.Name)); err != nil {
+		err = c.ds.DeleteConfigMap(c.namespace, types.GetConfigMapNameFromShareManagerName(sm.Name))
+		if err != nil && !datastore.ErrorIsNotFound(err) {
 			return errors.Wrapf(err, "failed to delete the configmap (recovery backend) for share manager %v", sm.Name)
 		}
 

diff --git a/controller/system_backup_controller.go b/controller/system_backup_controller.go
@@ -263,7 +263,12 @@ func (c *SystemBackupController) syncSystemBackup(key string) (err error) {
 		return nil
 	}
 
-	backupTargetClient, err := getBackupTargetClientDefault(c.ds)
+	backupTarget, err := c.ds.GetDefaultBackupTargetRO()
+	if err != nil {
+		return err
+	}
+
+	backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(c.ds, backupTarget)
 	if err != nil {
 		return err
 	}

diff --git a/controller/system_restore_controller.go b/controller/system_restore_controller.go
@@ -170,7 +170,12 @@ func (c *SystemRestoreController) syncSystemRestore(key string) (err error) {
 		return nil
 	}
 
-	backupTargetClient, err := getBackupTargetClientDefault(c.ds)
+	backupTarget, err := c.ds.GetDefaultBackupTargetRO()
+	if err != nil {
+		return err
+	}
+
+	backupTargetClient, err := newBackupTargetClientFromDefaultEngineImage(c.ds, backupTarget)
 	if err != nil {
 		return err
 	}