sftpd: add support for excluding virtual folders from user quota limit

Fixes drakkan#110
devopsmakers · May 1, 2020 · 3f75d46 · 3f75d46
1 parent 14c2a24
commit 3f75d46
Show file tree

Hide file tree

Showing 16 changed files with 340 additions and 139 deletions.
diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go
@@ -652,8 +652,9 @@ func validateVirtualFolders(user *User) error {
 				v.MappedPath, user.GetHomeDir())}
 		}
 		virtualFolders = append(virtualFolders, vfs.VirtualFolder{
-			VirtualPath: cleanedVPath,
-			MappedPath:  cleanedMPath,
+			VirtualPath:      cleanedVPath,
+			MappedPath:       cleanedMPath,
+			ExcludeFromQuota: v.ExcludeFromQuota,
 		})
 		for k, virtual := range mappedPaths {
 			if isMappedDirOverlapped(k, cleanedMPath) {

diff --git a/dataprovider/user.go b/dataprovider/user.go
@@ -191,6 +191,22 @@ func (u *User) GetPermissionsForPath(p string) []string {
 	return permissions
 }
 
+// IsFileExcludedFromQuota returns true if the file must be excluded from quota usage
+func (u *User) IsFileExcludedFromQuota(sftpPath string) bool {
+	if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != 0 {
+		return false
+	}
+	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(sftpPath))
+	for _, val := range dirsForPath {
+		for _, v := range u.VirtualFolders {
+			if v.VirtualPath == val {
+				return v.ExcludeFromQuota
+			}
+		}
+	}
+	return false
+}
+
 // AddVirtualDirs adds virtual folders, if defined, to the given files list
 func (u *User) AddVirtualDirs(list []os.FileInfo, sftpPath string) []os.FileInfo {
 	if len(u.VirtualFolders) == 0 {

diff --git a/docs/account.md b/docs/account.md
@@ -8,7 +8,7 @@ For each account, the following properties can be configured:
 - `status` 1 means "active", 0 "inactive". An inactive account cannot login.
 - `expiration_date` expiration date as unix timestamp in milliseconds. An expired account cannot login. 0 means no expiration.
 - `home_dir` the user cannot upload or download files outside this directory. Must be an absolute path. A local home directory is required for Cloud Storage Backends too: in this case it will store temporary files.
-- `virtual_folders` list of mappings between virtual SFTP/SCP paths and local filesystem paths outside the user home directory. The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory. The parent directory for the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login
+- `virtual_folders` list of mappings between virtual SFTP/SCP paths and local filesystem paths outside the user home directory. The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory. The parent directory for the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login. For each mapping you can configure if the folder will be included or not in user quota limit.
 - `uid`, `gid`. If SFTPGo runs as root system user then the created files and directories will be assigned to this system uid/gid. Ignored on windows or if SFTPGo runs as non root user: in this case files and directories for all SFTP users will be owned by the system user that runs SFTPGo.
 - `max_sessions` maximum concurrent sessions. 0 means unlimited.
 - `quota_size` maximum size allowed as bytes. 0 means unlimited.

diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go
@@ -183,7 +183,7 @@ func TestInitialization(t *testing.T) {
 	}
 	err = httpd.ReloadTLSCertificate()
 	if err != nil {
-		t.Error("realoding TLS Certificate must return nil error if no certificate is configured")
+		t.Error("reloading TLS Certificate must return nil error if no certificate is configured")
 	}
 }
 
@@ -628,8 +628,9 @@ func TestUpdateUser(t *testing.T) {
 		MappedPath:  filepath.Join(os.TempDir(), "mapped_dir1"),
 	})
 	user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
-		VirtualPath: "/vdir12/subdir",
-		MappedPath:  filepath.Join(os.TempDir(), "mapped_dir2"),
+		VirtualPath:      "/vdir12/subdir",
+		MappedPath:       filepath.Join(os.TempDir(), "mapped_dir2"),
+		ExcludeFromQuota: true,
 	})
 	user, _, err = httpd.UpdateUser(user, http.StatusOK)
 	if err != nil {
@@ -1793,7 +1794,7 @@ func TestWebUserAddMock(t *testing.T) {
 	form.Set("expiration_date", "")
 	form.Set("permissions", "*")
 	form.Set("sub_dirs_permissions", " /subdir::list ,download ")
-	form.Set("virtual_folders", fmt.Sprintf(" /vdir:: %v ", mappedDir))
+	form.Set("virtual_folders", fmt.Sprintf(" /vdir:: %v ::1", mappedDir))
 	form.Set("allowed_extensions", "/dir1::.jpg,.png")
 	form.Set("denied_extensions", "/dir1::.zip")
 	b, contentType, _ := getMultipartFormData(form, "", "")
@@ -1899,10 +1900,7 @@ func TestWebUserAddMock(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusOK, rr.Code)
 	var users []dataprovider.User
-	err := render.DecodeJSON(rr.Body, &users)
-	if err != nil {
-		t.Errorf("Error decoding users: %v", err)
-	}
+	render.DecodeJSON(rr.Body, &users)
 	if len(users) != 1 {
 		t.Errorf("1 user is expected, actual: %v", len(users))
 	}
@@ -1928,7 +1926,7 @@ func TestWebUserAddMock(t *testing.T) {
 	}
 	vfolderFoumd := false
 	for _, v := range newUser.VirtualFolders {
-		if v.VirtualPath == "/vdir" && v.MappedPath == mappedDir {
+		if v.VirtualPath == "/vdir" && v.MappedPath == mappedDir && v.ExcludeFromQuota == true {
 			vfolderFoumd = true
 		}
 	}

diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml
@@ -2,7 +2,7 @@ openapi: 3.0.1
 info:
   title: SFTPGo
   description: 'SFTPGo REST API'
-  version: 1.8.4
+  version: 1.8.5
 
 servers:
 - url: /api/v1
@@ -1077,6 +1077,10 @@ components:
           type: string
         mapped_path:
           type: string
+        exclude_from_quota:
+          type: boolean
+          nullable: true
+          description: This folder will be excluded from user quota
       required:
         - virtual_path
         - mapped_path

diff --git a/httpd/web.go b/httpd/web.go
@@ -196,10 +196,17 @@ func getVirtualFoldersFromPostFields(r *http.Request) []vfs.VirtualFolder {
 		if strings.Contains(cleaned, "::") {
 			mapping := strings.Split(cleaned, "::")
 			if len(mapping) > 1 {
-				virtualFolders = append(virtualFolders, vfs.VirtualFolder{
+				vfolder := vfs.VirtualFolder{
 					VirtualPath: strings.TrimSpace(mapping[0]),
 					MappedPath:  strings.TrimSpace(mapping[1]),
-				})
+				}
+				if len(mapping) > 2 {
+					excludeFromQuota, err := strconv.Atoi(strings.TrimSpace(mapping[2]))
+					if err == nil {
+						vfolder.ExcludeFromQuota = (excludeFromQuota > 0)
+					}
+				}
+				virtualFolders = append(virtualFolders, vfolder)
 			}
 		}
 	}

diff --git a/scripts/README.md b/scripts/README.md
@@ -140,7 +140,7 @@ Output:
 Command:
 
 ```
-python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1::list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --denied-login-methods "" --fs local --virtual-folders "/vdir1::/tmp/mapped1" "/vdir2::/tmp/mapped2" --allowed-extensions "" --denied-extensions ""
+python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1::list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --denied-login-methods "" --fs local --virtual-folders "/vdir1::/tmp/mapped1" "/vdir2::/tmp/mapped2::1" --allowed-extensions "" --denied-extensions ""
 ```
 
 Output:
@@ -203,10 +203,12 @@ Output:
   "username": "test_username",
   "virtual_folders": [
     {
+      "exclude_from_quota": false,
       "mapped_path": "/tmp/mapped1",
       "virtual_path": "/vdir1"
     },
     {
+      "exclude_from_quota": true,
       "mapped_path": "/tmp/mapped2",
       "virtual_path": "/vdir2"
     }
@@ -265,10 +267,12 @@ Output:
     "username": "test_username",
     "virtual_folders": [
       {
+        "exclude_from_quota": false,
         "mapped_path": "/tmp/mapped1",
         "virtual_path": "/vdir1"
       },
       {
+        "exclude_from_quota": true,
         "mapped_path": "/tmp/mapped2",
         "virtual_path": "/vdir2"
       }

diff --git a/scripts/sftpgo_api_cli.py b/scripts/sftpgo_api_cli.py
@@ -110,12 +110,19 @@ def buildVirtualFolders(self, vfolders):
 			if '::' in f:
 				vpath = ''
 				mapped_path = ''
+				exclude_from_quota = False
 				values = f.split('::')
 				if len(values) > 1:
 					vpath = values[0]
 					mapped_path = values[1]
+				if len(values) > 2:
+					try:
+						exclude_from_quota = int(values[2]) > 0
+					except:
+						pass
 				if vpath and mapped_path:
-					result.append({"virtual_path":vpath, "mapped_path":mapped_path})
+					result.append({"virtual_path":vpath, "mapped_path":mapped_path,
+								"exclude_from_quota":exclude_from_quota})
 		return result
 
 	def buildPermissions(self, root_perms, subdirs_perms):
@@ -508,7 +515,8 @@ def addCommonUserArguments(parser):
 	parser.add_argument('--subdirs-permissions', type=str, nargs='*', default=[], help='Permissions for subdirs. '
 					+'For example: "/somedir::list,download" "/otherdir/subdir::*" Default: %(default)s')
 	parser.add_argument('--virtual-folders', type=str, nargs='*', default=[], help='Virtual folder mapping. For example: '
-					+'"/vpath::/home/adir" "/vpath::C:\adir", ignored for non local filesystems. Default: %(default)s')
+					+'"/vpath::/home/adir" "/vpath::C:\adir::1". If the optional third argument is > 0 the virtual '
+					+'folder will be excluded from user quota. Ignored for non local filesystems. Default: %(default)s')
 	parser.add_argument('-U', '--upload-bandwidth', type=int, default=0,
 					help='Maximum upload bandwidth as KB/s, 0 means unlimited. Default: %(default)s')
 	parser.add_argument('-D', '--download-bandwidth', type=int, default=0,

diff --git a/sftpd/handler.go b/sftpd/handler.go
@@ -75,25 +75,26 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
 	c.Log(logger.LevelDebug, logSender, "fileread requested for path: %#v", p)
 
 	transfer := Transfer{
-		file:           file,
-		readerAt:       r,
-		writerAt:       nil,
-		cancelFn:       cancelFn,
-		path:           p,
-		start:          time.Now(),
-		bytesSent:      0,
-		bytesReceived:  0,
-		user:           c.User,
-		connectionID:   c.ID,
-		transferType:   transferDownload,
-		lastActivity:   time.Now(),
-		isNewFile:      false,
-		protocol:       c.protocol,
-		transferError:  nil,
-		isFinished:     false,
-		minWriteOffset: 0,
-		expectedSize:   fi.Size(),
-		lock:           new(sync.Mutex),
+		file:                file,
+		readerAt:            r,
+		writerAt:            nil,
+		cancelFn:            cancelFn,
+		path:                p,
+		start:               time.Now(),
+		bytesSent:           0,
+		bytesReceived:       0,
+		user:                c.User,
+		connectionID:        c.ID,
+		transferType:        transferDownload,
+		lastActivity:        time.Now(),
+		isNewFile:           false,
+		protocol:            c.protocol,
+		transferError:       nil,
+		isFinished:          false,
+		minWriteOffset:      0,
+		expectedSize:        fi.Size(),
+		isExcludedFromQuota: c.User.IsFileExcludedFromQuota(request.Filepath),
+		lock:                new(sync.Mutex),
 	}
 	addTransfer(&transfer)
 	return &transfer, nil
@@ -123,7 +124,7 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 		if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(request.Filepath)) {
 			return nil, sftp.ErrSSHFxPermissionDenied
 		}
-		return c.handleSFTPUploadToNewFile(p, filePath)
+		return c.handleSFTPUploadToNewFile(p, filePath, c.User.IsFileExcludedFromQuota(request.Filepath))
 	}
 
 	if statErr != nil {
@@ -141,7 +142,8 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 		return nil, sftp.ErrSSHFxPermissionDenied
 	}
 
-	return c.handleSFTPUploadToExistingFile(request.Pflags(), p, filePath, stat.Size())
+	return c.handleSFTPUploadToExistingFile(request.Pflags(), p, filePath, stat.Size(),
+		c.User.IsFileExcludedFromQuota(request.Filepath))
 }
 
 // Filecmd hander for basic SFTP system calls related to files, but not anything to do with reading
@@ -437,14 +439,16 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err
 
 	logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
 	if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-		dataprovider.UpdateUserQuota(dataProvider, c.User, -1, -size, false) //nolint:errcheck
+		if !c.User.IsFileExcludedFromQuota(request.Filepath) {
+			dataprovider.UpdateUserQuota(dataProvider, c.User, -1, -size, false) //nolint:errcheck
+		}
 	}
 	go executeAction(newActionNotification(c.User, operationDelete, filePath, "", "", fi.Size(), nil)) //nolint:errcheck
 
 	return sftp.ErrSSHFxOk
 }
 
-func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io.WriterAt, error) {
+func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string, isExcludedFromQuota bool) (io.WriterAt, error) {
 	if !c.hasSpace(true) {
 		c.Log(logger.LevelInfo, logSender, "denying file write due to space limit")
 		return nil, sftp.ErrSSHFxFailure
@@ -459,31 +463,32 @@ func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io.
 	vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
 
 	transfer := Transfer{
-		file:           file,
-		writerAt:       w,
-		readerAt:       nil,
-		cancelFn:       cancelFn,
-		path:           requestPath,
-		start:          time.Now(),
-		bytesSent:      0,
-		bytesReceived:  0,
-		user:           c.User,
-		connectionID:   c.ID,
-		transferType:   transferUpload,
-		lastActivity:   time.Now(),
-		isNewFile:      true,
-		protocol:       c.protocol,
-		transferError:  nil,
-		isFinished:     false,
-		minWriteOffset: 0,
-		lock:           new(sync.Mutex),
+		file:                file,
+		writerAt:            w,
+		readerAt:            nil,
+		cancelFn:            cancelFn,
+		path:                requestPath,
+		start:               time.Now(),
+		bytesSent:           0,
+		bytesReceived:       0,
+		user:                c.User,
+		connectionID:        c.ID,
+		transferType:        transferUpload,
+		lastActivity:        time.Now(),
+		isNewFile:           true,
+		protocol:            c.protocol,
+		transferError:       nil,
+		isFinished:          false,
+		minWriteOffset:      0,
+		isExcludedFromQuota: isExcludedFromQuota,
+		lock:                new(sync.Mutex),
 	}
 	addTransfer(&transfer)
 	return &transfer, nil
 }
 
 func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, requestPath, filePath string,
-	fileSize int64) (io.WriterAt, error) {
+	fileSize int64, isExcludedFromQuota bool) (io.WriterAt, error) {
 	var err error
 	if !c.hasSpace(false) {
 		c.Log(logger.LevelInfo, logSender, "denying file write due to space limit")
@@ -520,7 +525,9 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
 		minWriteOffset = fileSize
 	} else {
 		if vfs.IsLocalOsFs(c.fs) {
-			dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false) //nolint:errcheck
+			if !isExcludedFromQuota {
+				dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false) //nolint:errcheck
+			}
 		} else {
 			initialSize = fileSize
 		}
@@ -529,25 +536,26 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
 	vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
 
 	transfer := Transfer{
-		file:           file,
-		writerAt:       w,
-		readerAt:       nil,
-		cancelFn:       cancelFn,
-		path:           requestPath,
-		start:          time.Now(),
-		bytesSent:      0,
-		bytesReceived:  0,
-		user:           c.User,
-		connectionID:   c.ID,
-		transferType:   transferUpload,
-		lastActivity:   time.Now(),
-		isNewFile:      false,
-		protocol:       c.protocol,
-		transferError:  nil,
-		isFinished:     false,
-		minWriteOffset: minWriteOffset,
-		initialSize:    initialSize,
-		lock:           new(sync.Mutex),
+		file:                file,
+		writerAt:            w,
+		readerAt:            nil,
+		cancelFn:            cancelFn,
+		path:                requestPath,
+		start:               time.Now(),
+		bytesSent:           0,
+		bytesReceived:       0,
+		user:                c.User,
+		connectionID:        c.ID,
+		transferType:        transferUpload,
+		lastActivity:        time.Now(),
+		isNewFile:           false,
+		protocol:            c.protocol,
+		transferError:       nil,
+		isFinished:          false,
+		minWriteOffset:      minWriteOffset,
+		initialSize:         initialSize,
+		isExcludedFromQuota: isExcludedFromQuota,
+		lock:                new(sync.Mutex),
 	}
 	addTransfer(&transfer)
 	return &transfer, nil