add Azure Blob support

devopsmakers · Oct 25, 2020 · 5ff8f75 · 5ff8f75
1 parent db7e81e
commit 5ff8f75
Show file tree

Hide file tree

Showing 31 changed files with 1,502 additions and 157 deletions.
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go)
 
 Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support, written in Go.
-It can serve local filesystem, S3 or Google Cloud Storage.
+It can serve local filesystem, S3 (compatible) Object Storage, Google Cloud Storage and Azure Blob Storage.
 
 ## Features
 
@@ -173,6 +173,10 @@ Each user can be mapped to the whole bucket or to a bucket virtual folder. This
 
 Each user can be mapped with a Google Cloud Storage bucket or a bucket virtual folder. This way, the mapped bucket/virtual folder is exposed over SFTP/SCP/FTP/WebDAV. More information about Google Cloud Storage integration can be found [here](./docs/google-cloud-storage.md).
 
+### Azure Blob Storage backend
+
+Each user can be mapped with an Azure Blob Storage container or a container virtual folder. This way, the mapped container/virtual folder is exposed over SFTP/SCP/FTP/WebDAV. More information about Azure Blob Storage integration can be found [here](./docs/azure-blob-storage.md).
+
 ### Other Storage backends
 
 Adding new storage backends is quite easy:

diff --git a/common/actions.go b/common/actions.go
@@ -84,6 +84,13 @@ func newActionNotification(
 		endpoint = user.FsConfig.S3Config.Endpoint
 	} else if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider {
 		bucket = user.FsConfig.GCSConfig.Bucket
+	} else if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider {
+		bucket = user.FsConfig.AzBlobConfig.Container
+		if user.FsConfig.AzBlobConfig.SASURL != "" {
+			endpoint = user.FsConfig.AzBlobConfig.SASURL
+		} else {
+			endpoint = user.FsConfig.AzBlobConfig.Endpoint
+		}
 	}
 
 	if err == ErrQuotaExceeded {

diff --git a/common/actions_test.go b/common/actions_test.go
@@ -28,6 +28,11 @@ func TestNewActionNotification(t *testing.T) {
 	user.FsConfig.GCSConfig = vfs.GCSFsConfig{
 		Bucket: "gcsbucket",
 	}
+	user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{
+		Container: "azcontainer",
+		SASURL:    "azsasurl",
+		Endpoint:  "azendpoint",
+	}
 	a := newActionNotification(user, operationDownload, "path", "target", "", ProtocolSFTP, 123, errors.New("fake error"))
 	assert.Equal(t, user.Username, a.Username)
 	assert.Equal(t, 0, len(a.Bucket))
@@ -45,6 +50,18 @@ func TestNewActionNotification(t *testing.T) {
 	assert.Equal(t, "gcsbucket", a.Bucket)
 	assert.Equal(t, 0, len(a.Endpoint))
 	assert.Equal(t, 2, a.Status)
+
+	user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSCP, 123, nil)
+	assert.Equal(t, "azcontainer", a.Bucket)
+	assert.Equal(t, "azsasurl", a.Endpoint)
+	assert.Equal(t, 1, a.Status)
+
+	user.FsConfig.AzBlobConfig.SASURL = ""
+	a = newActionNotification(user, operationDownload, "path", "target", "", ProtocolSCP, 123, nil)
+	assert.Equal(t, "azcontainer", a.Bucket)
+	assert.Equal(t, "azendpoint", a.Endpoint)
+	assert.Equal(t, 1, a.Status)
 }
 
 func TestActionHTTP(t *testing.T) {

diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go
@@ -995,7 +995,7 @@ func validateFilesystemConfig(user *User) error {
 		if err != nil {
 			return &ValidationError{err: fmt.Sprintf("could not validate s3config: %v", err)}
 		}
-		if len(user.FsConfig.S3Config.AccessSecret) > 0 {
+		if user.FsConfig.S3Config.AccessSecret != "" {
 			vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$")
 			if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 {
 				accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret)
@@ -1012,10 +1012,27 @@ func validateFilesystemConfig(user *User) error {
 			return &ValidationError{err: fmt.Sprintf("could not validate GCS config: %v", err)}
 		}
 		return nil
+	} else if user.FsConfig.Provider == AzureBlobFilesystemProvider {
+		err := vfs.ValidateAzBlobFsConfig(&user.FsConfig.AzBlobConfig)
+		if err != nil {
+			return &ValidationError{err: fmt.Sprintf("could not validate Azure Blob config: %v", err)}
+		}
+		if user.FsConfig.AzBlobConfig.AccountKey != "" {
+			vals := strings.Split(user.FsConfig.AzBlobConfig.AccountKey, "$")
+			if !strings.HasPrefix(user.FsConfig.AzBlobConfig.AccountKey, "$aes$") || len(vals) != 4 {
+				accountKey, err := utils.EncryptData(user.FsConfig.AzBlobConfig.AccountKey)
+				if err != nil {
+					return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)}
+				}
+				user.FsConfig.AzBlobConfig.AccountKey = accountKey
+			}
+		}
+		return nil
 	}
 	user.FsConfig.Provider = LocalFilesystemProvider
 	user.FsConfig.S3Config = vfs.S3FsConfig{}
 	user.FsConfig.GCSConfig = vfs.GCSFsConfig{}
+	user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{}
 	return nil
 }
 
@@ -1248,6 +1265,8 @@ func HideUserSensitiveData(user *User) User {
 		user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret)
 	} else if user.FsConfig.Provider == GCSFilesystemProvider {
 		user.FsConfig.GCSConfig.Credentials = nil
+	} else if user.FsConfig.Provider == AzureBlobFilesystemProvider {
+		user.FsConfig.AzBlobConfig.AccountKey = utils.RemoveDecryptionKey(user.FsConfig.AzBlobConfig.AccountKey)
 	}
 	return *user
 }

diff --git a/dataprovider/user.go b/dataprovider/user.go
@@ -124,16 +124,18 @@ type FilesystemProvider int
 
 // supported values for FilesystemProvider
 const (
-	LocalFilesystemProvider FilesystemProvider = iota // Local
-	S3FilesystemProvider                              // Amazon S3 compatible
-	GCSFilesystemProvider                             // Google Cloud Storage
+	LocalFilesystemProvider     FilesystemProvider = iota // Local
+	S3FilesystemProvider                                  // Amazon S3 compatible
+	GCSFilesystemProvider                                 // Google Cloud Storage
+	AzureBlobFilesystemProvider                           // Azure Blob Storage
 )
 
 // Filesystem defines cloud storage filesystem details
 type Filesystem struct {
-	Provider  FilesystemProvider `json:"provider"`
-	S3Config  vfs.S3FsConfig     `json:"s3config,omitempty"`
-	GCSConfig vfs.GCSFsConfig    `json:"gcsconfig,omitempty"`
+	Provider     FilesystemProvider `json:"provider"`
+	S3Config     vfs.S3FsConfig     `json:"s3config,omitempty"`
+	GCSConfig    vfs.GCSFsConfig    `json:"gcsconfig,omitempty"`
+	AzBlobConfig vfs.AzBlobFsConfig `json:"azblobconfig,omitempty"`
 }
 
 // User defines a SFTPGo user
@@ -196,6 +198,8 @@ func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) {
 		config := u.FsConfig.GCSConfig
 		config.CredentialFile = u.getGCSCredentialsFilePath()
 		return vfs.NewGCSFs(connectionID, u.GetHomeDir(), config)
+	} else if u.FsConfig.Provider == AzureBlobFilesystemProvider {
+		return vfs.NewAzBlobFs(connectionID, u.GetHomeDir(), u.FsConfig.AzBlobConfig)
 	}
 	return vfs.NewOsFs(connectionID, u.GetHomeDir(), u.VirtualFolders), nil
 }
@@ -626,6 +630,8 @@ func (u *User) GetInfoString() string {
 		result += "Storage: S3 "
 	} else if u.FsConfig.Provider == GCSFilesystemProvider {
 		result += "Storage: GCS "
+	} else if u.FsConfig.Provider == AzureBlobFilesystemProvider {
+		result += "Storage: Azure "
 	}
 	if len(u.PublicKeys) > 0 {
 		result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys))
@@ -725,6 +731,17 @@ func (u *User) getACopy() User {
 			StorageClass:         u.FsConfig.GCSConfig.StorageClass,
 			KeyPrefix:            u.FsConfig.GCSConfig.KeyPrefix,
 		},
+		AzBlobConfig: vfs.AzBlobFsConfig{
+			Container:         u.FsConfig.AzBlobConfig.Container,
+			AccountName:       u.FsConfig.AzBlobConfig.AccountName,
+			AccountKey:        u.FsConfig.AzBlobConfig.AccountKey,
+			Endpoint:          u.FsConfig.AzBlobConfig.Endpoint,
+			SASURL:            u.FsConfig.AzBlobConfig.SASURL,
+			KeyPrefix:         u.FsConfig.AzBlobConfig.KeyPrefix,
+			UploadPartSize:    u.FsConfig.AzBlobConfig.UploadPartSize,
+			UploadConcurrency: u.FsConfig.AzBlobConfig.UploadConcurrency,
+			UseEmulator:       u.FsConfig.AzBlobConfig.UseEmulator,
+		},
 	}
 
 	return User{

diff --git a/docs/account.md b/docs/account.md
@@ -45,7 +45,7 @@ For each account, the following properties can be configured:
   - `allowed_extensions`, list of, case insensitive, allowed files extension. Shell like expansion is not supported so you have to specify `.jpg` and not `*.jpg`. Any file that does not end with this suffix will be denied
   - `denied_extensions`, list of, case insensitive, denied files extension. Denied file extensions are evaluated before the allowed ones
   - `path`, SFTP/SCP path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths `/` and `/sub` then the filters for `/` are applied for any file outside the `/sub` directory
-- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported
+- `fs_provider`, filesystem to serve via SFTP. Local filesystem, S3 Compatible Object Storage, Google Cloud Storage and Azure Blob Storage are supported
 - `s3_bucket`, required for S3 filesystem
 - `s3_region`, required for S3 filesystem. Must match the region for your bucket. You can find here the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example if your bucket is at `Frankfurt` you have to set the region to `eu-central-1`
 - `s3_access_key`
@@ -60,6 +60,15 @@ For each account, the following properties can be configured:
 - `gcs_automatic_credentials`, integer. Set to 1 to use Application Default Credentials strategy or set to 0 to use explicit credentials via `gcs_credentials`
 - `gcs_storage_class`
 - `gcs_key_prefix`, allows to restrict access to the folder identified by this prefix and its contents
+- `az_container`, Azure Blob Storage container
+- `az_account_name`, Azure account name. leave blank to use SAS URL
+- `az_account_key`, Azure account key. leave blank to use SAS URL. If provided it is stored encrypted (AES-256-GCM)
+- `az_sas_url`, Azure shared access signature URL
+- `az_endpoint`, Default is "blob.core.windows.net". If you use the emulator the endpoint must include the protocol, for example "http://127.0.0.1:10000"
+- `az_upload_part_size`, the buffer size for multipart uploads (MB). Zero means the default (4 MB)
+- `az_upload_concurrency`,  how many parts are uploaded in parallel. Zero means the default (2)
+- `az_key_prefix`,  allows to restrict access to the folder identified by this prefix and its contents
+- `az_use_emulator`, boolean
 
 These properties are stored inside the data provider.
 

diff --git a/docs/azure-blob-storage.md b/docs/azure-blob-storage.md
@@ -0,0 +1,20 @@
+# Azure Blob Storage backend
+
+To connect SFTPGo to Azure Blob Storage, you need to specify the access credentials. Azure Blob Storage has different options for credentials, we support:
+
+1. Providing an account name and account key.
+2. Providing a shared access signature (SAS).
+
+If you authenticate using account and key you also need to specify a container. The endpoint can generally be left blank, the default is `blob.core.windows.net`.
+
+If you provide a SAS URL the container is optional and if given it must match the one inside the shared access signature.
+
+If you want to connect to an emulator such as [Azurite](https://github.com/Azure/Azurite) you need to provide the account name/key pair and an endpoint prefixed with the protocol, for example `http://127.0.0.1:10000`.
+
+Specifying a different `key_prefix`, you can assign different "folders" of the same container to different users. This is similar to a chroot directory for local filesystem. Each SFTPGo user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
+
+For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the client and SFTPGo is greater than the upload bandwidth between SFTPGo and the Azure Blob service then the client should wait for the last parts to be uploaded to Azure after finishing uploading the file to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
+
+The configured container must exist.
+
+This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.
diff --git a/docs/build-from-source.md b/docs/build-from-source.md
@@ -14,6 +14,7 @@ The following build tags are available:
 
 - `nogcs`, disable Google Cloud Storage backend, default enabled
 - `nos3`, disable S3 Compabible Object Storage backends, default enabled
+- `noazblob`, disable Azure Blob Storage backend, default enabled
 - `nobolt`, disable Bolt data provider, default enabled
 - `nomysql`, disable MySQL data provider, default enabled
 - `nopgsql`, disable PostgreSQL data provider, default enabled

diff --git a/docs/custom-actions.md b/docs/custom-actions.md
@@ -23,9 +23,9 @@ The external program can also read the following environment variables:
 - `SFTPGO_ACTION_TARGET`, non-empty for `rename` `SFTPGO_ACTION`
 - `SFTPGO_ACTION_SSH_CMD`, non-empty for `ssh_cmd` `SFTPGO_ACTION`
 - `SFTPGO_ACTION_FILE_SIZE`, non-empty for `upload`, `download` and `delete` `SFTPGO_ACTION`
-- `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
-- `SFTPGO_ACTION_BUCKET`, non-empty for S3 and GCS backends
-- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3 backend if configured
+- `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend
+- `SFTPGO_ACTION_BUCKET`, non-empty for S3, GCS and Azure backends
+- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3 and Azure backend if configured. For Azure this is the SAS URL, if configured otherwise the endpoint
 - `SFTPGO_ACTION_STATUS`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
 - `SFTPGO_ACTION_PROTOCOL`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`
 
@@ -40,9 +40,9 @@ If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. Th
 - `target_path`, not null for `rename` action
 - `ssh_cmd`, not null for `ssh_cmd` action
 - `file_size`, not null for `upload`, `download`, `delete` actions
-- `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend
-- `bucket`, not null for S3 and GCS backends
-- `endpoint`, not null for S3 backend if configured
+- `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend
+- `bucket`, not null for S3, GCS and Azure backends
+- `endpoint`, not null for S3 and Azure backend if configured. For Azure this is the SAS URL, if configured otherwise the endpoint
 - `status`, integer. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error
 - `protocol`, string. Possible values are `SSH`, `FTP`, `DAV`
 

diff --git a/docs/google-cloud-storage.md b/docs/google-cloud-storage.md
@@ -8,6 +8,4 @@ You can optionally specify a [storage class](https://cloud.google.com/storage/do
 
 The configured bucket must exist.
 
-Google Cloud Storage is exposed over HTTPS so if you are running SFTPGo as docker image please be sure to uncomment the line that installs `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
-
 This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.
diff --git a/docs/s3.md b/docs/s3.md
@@ -10,13 +10,11 @@ AWS SDK has different options for credentials. [More Detail](https://docs.aws.am
 
 So, you need to provide access keys to activate option 1, or leave them blank to use the other ways to specify credentials.
 
-Most S3 backends require HTTPS connections so if you are running SFTPGo as docker image please be sure to uncomment the line that installs `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.
-
 Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
 
 SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3.
 
-For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the SFTP client and SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then the SFTP client have to wait for the upload of the last parts to S3 after it ends the file upload to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
+For multipart uploads you can customize the parts size and the upload concurrency. Please note that if the upload bandwidth between the client and SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then the client should wait for the last parts to be uploaded to S3 after finishing uploading the file to SFTPGo, and it may time out. Keep this in mind if you customize these parameters.
 
 The configured bucket must exist.
 
@@ -32,7 +30,7 @@ Some SFTP commands don't work over S3:
 Other notes:
 
 - `rename` is a two step operation: server-side copy and then deletion. So, it is not atomic as for local filesystem.
-- We don't support renaming non empty directories since we should rename all the contents too and this could take a long time: think about directories with thousands of files; for each file we should do an AWS API call.
+- We don't support renaming non empty directories since we should rename all the contents too and this could take a long time: think about directories with thousands of files: for each file we should do an AWS API call.
 - For server side encryption, you have to configure the mapped bucket to automatically encrypt objects.
 - A local home directory is still required to store temporary files.
 - Clients that require advanced filesystem-like features such as `sshfs` are not supported.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,6 +8,4 @@ You can optionally specify a [storage class](https://cloud.google.com/storage/do

		The configured bucket must exist.

		Google Cloud Storage is exposed over HTTPS so if you are running SFTPGo as docker image please be sure to uncomment the line that installs `ca-certificates`, inside your `Dockerfile`, to be able to properly verify certificate authorities.

		This backend is very similar to the [S3](./s3.md) backend, and it has the same limitations.