Merge pull request #2 from hervenicol/retrymount

workaround "Error creating mount directory" errors at container startup
hervenicol · Aug 20, 2021 · 8927730 · 8927730
2 parents 57bee29 + 43de2fb
commit 8927730
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## v0.9.0
 * luks encryption support
+* fix (retry) mount error "mkdir - file exists"
 
 ## v0.8.0
 * fix did not umount when volume is broken

diff --git a/plugin.go b/plugin.go
@@ -260,7 +260,7 @@ func (d plugin) Mount(r *volume.MountRequest) (*volume.MountResponse, error) {
 
 	// Is it encrypted?
 	if result, err := isLuks(physdev); result == true {
-		logger.Debugf("Encrypted volume - using key file \"%s\"", d.config.EncryptionKey)
+		logger.Debugf("Encrypted volume - using key file '%s'", d.config.EncryptionKey)
 		// If yes, we must have a passphrase.
 		if d.config.EncryptionKey == "" {
 			logger.Errorf("Device %s is encrypted, and I have no pass to decrypt it.", physdev)
@@ -280,8 +280,8 @@ func (d plugin) Mount(r *volume.MountRequest) (*volume.MountResponse, error) {
 	}
 
 
-    //
-    // Check filesystem and format if needed
+	//
+	// Check filesystem and format if needed
 
 	fsType, err := getFilesystemType(dev)
 	if err != nil {
@@ -310,8 +310,10 @@ func (d plugin) Mount(r *volume.MountRequest) (*volume.MountResponse, error) {
 	// Mount device
 
 	path := filepath.Join(d.config.MountDir, r.Name)
-	if err = os.MkdirAll(path, 0700); err != nil {
-		logger.WithError(err).Error("Error creating mount directory")
+
+	err = createMountDir(path)
+	if err != nil {
+		logger.WithError(err).Errorf("Error creating mount directory %s", path)
 		return nil, err
 	}
 

diff --git a/util.go b/util.go
@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 	"bufio"
+	"syscall"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
@@ -110,7 +111,7 @@ func getLuksInfo(mountPath string) (string, string, string, error) {
 	}
 
 	// All went well, here is the retrieved info
-	logger.Debugf("Mount found for %s - device %s - luks name %s - base device %s\n", mountPath, mountDevice, luksName, baseDevice)
+	logger.Debugf("Mount found for '%s' - device '%s' - luks name '%s' - base device '%s'", mountPath, mountDevice, luksName, baseDevice)
 	return mountDevice, luksName, baseDevice, nil
 }
 
@@ -292,3 +293,35 @@ func isDirectoryPresent(path string) (bool, error) {
 		return stat.IsDir(), nil
 	}
 }
+
+func createMountDir(path string) (error) {
+	// Sometimes mkdir fails, and I've observed it is a symptom of a bug
+	// where volume is half-mounted (?)
+	// this can be solved with umount
+	// (anyway the volume should not be mounted at this point)
+
+	// as I suspect this "half-mounted" problem comes from a race condition
+	// where unmount of the previous container and mount of the new container
+	// may be too fast (or maybe at the same time?),
+	// I prefer to wait a bit before retrying the unmount.
+
+	logger := log.WithFields(log.Fields{"action": "createMountDir"})
+	sleep := 1 * time.Second
+	for retry := 0; retry < 3; retry++ {
+
+		// If mkdir is OK, proceed to next step
+		if err := os.MkdirAll(path, 0700); err == nil {
+			return nil
+		}
+
+		// exponential backoff
+		time.Sleep(sleep)
+		sleep = sleep * 2
+
+		err := syscall.Unmount(path, 0)
+		if err != nil {
+			logger.WithError(err).Errorf("Error unmount %s", path)
+		}
+	}
+	return fmt.Errorf("Failed creating directory %s", path)
+}