gcs: add support for automatic credentials

We can now also support implicit credentials using the Application Default Credentials strategy
devopsmakers · Feb 19, 2020 · ae8ed75 · ae8ed75
1 parent c8cc81c
commit ae8ed75
Show file tree

Hide file tree

Showing 15 changed files with 211 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -587,7 +587,7 @@ Other notes:
 
 Each user can be mapped with a Google Cloud Storage bucket or a bucket virtual folder, this way the mapped bucket/virtual folder is exposed over SFTP/SCP. This backend is very similar to the S3 backend and it has the same limitations.
 
-To connect SFTPGo to Google Cloud Storage you need a credentials file that you can obtain from the Google Cloud Console, take a look at the "Setting up authentication" section [here](https://cloud.google.com/storage/docs/reference/libraries) for details.
+To connect SFTPGo to Google Cloud Storage you can use use the Application Default Credentials (ADC) strategy to try to find your application's credentials automatically or you can explicitly provide a JSON credentials file that you can obtain from the Google Cloud Console, take a look [here](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application) for details.
 
 You can optionally specify a [storage class](https://cloud.google.com/storage/docs/storage-classes) too, leave blank to use the default storage class.
 
@@ -618,29 +618,30 @@ Usage:
   sftpgo portable [flags]
 
 Flags:
-  -C, --advertise-credentials         If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record
-  -S, --advertise-service             Advertise SFTP service using multicast DNS (default true)
-  -d, --directory string              Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
-  -f, --fs-provider int               0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+  -C, --advertise-credentials           If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record
+  -S, --advertise-service               Advertise SFTP service using multicast DNS (default true)
+  -d, --directory string                Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
+  -f, --fs-provider int                 0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+      --gcs-automatic-credentials int   0 means explicit credentials using a JSON credentials file, 1 automatic (default 1)
       --gcs-bucket string
-      --gcs-credentials-file string   Google Cloud Storage JSON credentials file
-      --gcs-key-prefix string         Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --gcs-credentials-file string     Google Cloud Storage JSON credentials file
+      --gcs-key-prefix string           Allows to restrict access to the virtual folder identified by this prefix and its contents
       --gcs-storage-class string
-  -h, --help                          help for portable
-  -l, --log-file-path string          Leave empty to disable logging
-  -p, --password string               Leave empty to use an auto generated value
-  -g, --permissions strings           User's permissions. "*" means any permission (default [list,download])
+  -h, --help                            help for portable
+  -l, --log-file-path string            Leave empty to disable logging
+  -p, --password string                 Leave empty to use an auto generated value
+  -g, --permissions strings             User's permissions. "*" means any permission (default [list,download])
   -k, --public-key strings
       --s3-access-key string
       --s3-access-secret string
       --s3-bucket string
       --s3-endpoint string
-      --s3-key-prefix string          Allows to restrict access to the virtual folder identified by this prefix and its contents
+      --s3-key-prefix string            Allows to restrict access to the virtual folder identified by this prefix and its contents
       --s3-region string
       --s3-storage-class string
-  -s, --sftpd-port int                0 means a random non privileged port
-  -c, --ssh-commands strings          SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
-  -u, --username string               Leave empty to use an auto generated value
+  -s, --sftpd-port int                  0 means a random non privileged port
+  -c, --ssh-commands strings            SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
+  -u, --username string                 Leave empty to use an auto generated value
 ```
 
 In portable mode SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
@@ -696,6 +697,7 @@ For each account the following properties can be configured:
 - `s3_key_prefix`, allows to restrict access to the virtual folder identified by this prefix and its contents
 - `gcs_bucket`, required for GCS filesystem
 - `gcs_credentials`, Google Cloud Storage JSON credentials base64 encoded
+- `gcs_automatic_credentials`, integer. Set to 1 to use Application Default Credentials strategy or set to 0 to use explicit credentials via `gcs_credentials`
 - `gcs_storage_class`
 - `gcs_key_prefix`, allows to restrict access to the virtual folder identified by this prefix and its contents
 

diff --git a/cmd/portable.go b/cmd/portable.go
@@ -35,6 +35,7 @@ var (
 	portableS3KeyPrefix          string
 	portableGCSBucket            string
 	portableGCSCredentialsFile   string
+	portableGCSAutoCredentials   int
 	portableGCSStorageClass      string
 	portableGCSKeyPrefix         string
 	portableCmd                  = &cobra.Command{
@@ -48,12 +49,16 @@ Please take a look at the usage below to customize the serving parameters`,
 		Run: func(cmd *cobra.Command, args []string) {
 			portableDir := directoryToServe
 			if !filepath.IsAbs(portableDir) {
-				portableDir, _ = filepath.Abs(portableDir)
+				if portableFsProvider == 0 {
+					portableDir, _ = filepath.Abs(portableDir)
+				} else {
+					portableDir = os.TempDir()
+				}
 			}
 			permissions := make(map[string][]string)
 			permissions["/"] = portablePermissions
 			portableGCSCredentials := ""
-			if portableFsProvider == 2 {
+			if portableFsProvider == 2 && len(portableGCSCredentialsFile) > 0 {
 				fi, err := os.Stat(portableGCSCredentialsFile)
 				if err != nil {
 					fmt.Printf("Invalid GCS credentials file: %v\n", err)
@@ -69,6 +74,7 @@ Please take a look at the usage below to customize the serving parameters`,
 					fmt.Printf("Unable to read credentials file: %v\n", err)
 				}
 				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
+				portableGCSAutoCredentials = 0
 			}
 			service := service.Service{
 				ConfigDir:     defaultConfigDir,
@@ -100,10 +106,11 @@ Please take a look at the usage below to customize the serving parameters`,
 							KeyPrefix:    portableS3KeyPrefix,
 						},
 						GCSConfig: vfs.GCSFsConfig{
-							Bucket:       portableGCSBucket,
-							Credentials:  portableGCSCredentials,
-							StorageClass: portableGCSStorageClass,
-							KeyPrefix:    portableGCSKeyPrefix,
+							Bucket:               portableGCSBucket,
+							Credentials:          portableGCSCredentials,
+							AutomaticCredentials: portableGCSAutoCredentials,
+							StorageClass:         portableGCSStorageClass,
+							KeyPrefix:            portableGCSKeyPrefix,
 						},
 					},
 				},
@@ -147,5 +154,7 @@ func init() {
 	portableCmd.Flags().StringVar(&portableGCSKeyPrefix, "gcs-key-prefix", "", "Allows to restrict access to the virtual folder "+
 		"identified by this prefix and its contents")
 	portableCmd.Flags().StringVar(&portableGCSCredentialsFile, "gcs-credentials-file", "", "Google Cloud Storage JSON credentials file")
+	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, "0 means explicit credentials using a JSON "+
+		"credentials file, 1 automatic")
 	rootCmd.AddCommand(portableCmd)
 }
diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go
@@ -813,6 +813,9 @@ func addCredentialsToUser(user *User) error {
 	if user.FsConfig.Provider != 2 {
 		return nil
 	}
+	if user.FsConfig.GCSConfig.AutomaticCredentials > 0 {
+		return nil
+	}
 	cred, err := ioutil.ReadFile(user.getGCSCredentialsFilePath())
 	if err != nil {
 		return err

diff --git a/dataprovider/user.go b/dataprovider/user.go
@@ -419,10 +419,11 @@ func (u *User) getACopy() User {
 			KeyPrefix:    u.FsConfig.S3Config.KeyPrefix,
 		},
 		GCSConfig: vfs.GCSFsConfig{
-			Bucket:         u.FsConfig.GCSConfig.Bucket,
-			CredentialFile: u.FsConfig.GCSConfig.CredentialFile,
-			StorageClass:   u.FsConfig.GCSConfig.StorageClass,
-			KeyPrefix:      u.FsConfig.GCSConfig.KeyPrefix,
+			Bucket:               u.FsConfig.GCSConfig.Bucket,
+			CredentialFile:       u.FsConfig.GCSConfig.CredentialFile,
+			AutomaticCredentials: u.FsConfig.GCSConfig.AutomaticCredentials,
+			StorageClass:         u.FsConfig.GCSConfig.StorageClass,
+			KeyPrefix:            u.FsConfig.GCSConfig.KeyPrefix,
 		},
 	}
 

diff --git a/httpd/api_utils.go b/httpd/api_utils.go
@@ -462,6 +462,9 @@ func compareUserFsConfig(expected *dataprovider.User, actual *dataprovider.User)
 		expected.FsConfig.GCSConfig.KeyPrefix+"/" != actual.FsConfig.GCSConfig.KeyPrefix {
 		return errors.New("GCS key prefix mismatch")
 	}
+	if expected.FsConfig.GCSConfig.AutomaticCredentials != actual.FsConfig.GCSConfig.AutomaticCredentials {
+		return errors.New("GCS automatic credentials mismatch")
+	}
 	return nil
 }
 

diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go
@@ -352,6 +352,7 @@ func TestAddUserInvalidFsConfig(t *testing.T) {
 	}
 	u.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/"
 	u.FsConfig.GCSConfig.Credentials = ""
+	u.FsConfig.GCSConfig.AutomaticCredentials = 0
 	_, _, err = httpd.AddUser(u, http.StatusBadRequest)
 	if err != nil {
 		t.Errorf("unexpected error adding user with invalid fs config: %v", err)
@@ -519,6 +520,14 @@ func TestUserGCSConfig(t *testing.T) {
 	if err != nil {
 		t.Errorf("unable to add user: %v", err)
 	}
+	os.RemoveAll(credentialsPath)
+	os.MkdirAll(credentialsPath, 0700)
+	user.FsConfig.GCSConfig.Credentials = ""
+	user.FsConfig.GCSConfig.AutomaticCredentials = 1
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to update user: %v", err)
+	}
 	user.FsConfig.Provider = 1
 	user.FsConfig.S3Config.Bucket = "test1"
 	user.FsConfig.S3Config.Region = "us-east-1"
@@ -1937,6 +1946,26 @@ func TestWebUserGCSMock(t *testing.T) {
 	if updateUser.FsConfig.GCSConfig.KeyPrefix != user.FsConfig.GCSConfig.KeyPrefix {
 		t.Error("GCS key prefix mismatch")
 	}
+	form.Set("gcs_auto_credentials", "on")
+	b, contentType, _ = getMultipartFormData(form, "", "")
+	req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b)
+	req.Header.Set("Content-Type", contentType)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusSeeOther, rr.Code)
+	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr.Code)
+	err = render.DecodeJSON(rr.Body, &users)
+	if err != nil {
+		t.Errorf("Error decoding users: %v", err)
+	}
+	if len(users) != 1 {
+		t.Errorf("1 user is expected")
+	}
+	updateUser = users[0]
+	if updateUser.FsConfig.GCSConfig.AutomaticCredentials != 1 {
+		t.Error("GCS automatic credentials mismatch")
+	}
 	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusOK, rr.Code)

diff --git a/httpd/internal_test.go b/httpd/internal_test.go
@@ -286,8 +286,13 @@ func TestCompareUserFsConfig(t *testing.T) {
 		t.Errorf("S3 key prefix does not match")
 	}
 	expected.FsConfig.S3Config.KeyPrefix = ""
+}
+
+func TestCompareUserGCSConfig(t *testing.T) {
+	expected := &dataprovider.User{}
+	actual := &dataprovider.User{}
 	expected.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir"
-	err = compareUserFsConfig(expected, actual)
+	err := compareUserFsConfig(expected, actual)
 	if err == nil {
 		t.Errorf("GCS key prefix does not match")
 	}
@@ -304,6 +309,12 @@ func TestCompareUserFsConfig(t *testing.T) {
 		t.Errorf("GCS storage class does not match")
 	}
 	expected.FsConfig.GCSConfig.StorageClass = ""
+	expected.FsConfig.GCSConfig.AutomaticCredentials = 1
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("GCS automatic credentials does not match")
+	}
+	expected.FsConfig.GCSConfig.AutomaticCredentials = 0
 }
 
 func TestGCSWebInvalidFormFile(t *testing.T) {

diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml
@@ -2,7 +2,7 @@ openapi: 3.0.1
 info:
   title: SFTPGo
   description: 'SFTPGo REST API'
-  version: 1.7.0
+  version: 1.8.0
 
 servers:
 - url: /api/v1
@@ -987,6 +987,16 @@ components:
           type: string
           format: byte
           description: Google Cloud Storage JSON credentials base64 encoded. This field must be populated only when adding/updating an user. It will be always omitted, since there are sensitive data, when you search/get users. The credentials will be stored in the configured "credentials_path"
+        automatic_credentials:
+          type: integer
+          nullable: true
+          enum:
+            - 0
+            - 1
+          description: >
+            Automatic credentials:
+              * `0` - disabled, explicit credentials, using a JSON credentials file, must be provided. This is the default value if the field is null
+              * `1` - enabled, we try to use the Application Default Credentials (ADC) strategy to find your application's credentials
         storage_class:
           type: string
         key_prefix:

diff --git a/httpd/web.go b/httpd/web.go
@@ -246,6 +246,12 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er
 		fs.GCSConfig.Bucket = r.Form.Get("gcs_bucket")
 		fs.GCSConfig.StorageClass = r.Form.Get("gcs_storage_class")
 		fs.GCSConfig.KeyPrefix = r.Form.Get("gcs_key_prefix")
+		autoCredentials := r.Form.Get("gcs_auto_credentials")
+		if len(autoCredentials) > 0 {
+			fs.GCSConfig.AutomaticCredentials = 1
+		} else {
+			fs.GCSConfig.AutomaticCredentials = 0
+		}
 		credentials, _, err := r.FormFile("gcs_credential_file")
 		if err == http.ErrMissingFile {
 			return fs, nil
@@ -262,6 +268,7 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er
 			return fs, err
 		}
 		fs.GCSConfig.Credentials = base64.StdEncoding.EncodeToString(fileBytes)
+		fs.GCSConfig.AutomaticCredentials = 0
 	}
 	return fs, nil
 }