add support for the venerable FTP protocol

Fixes drakkan#46
devopsmakers · Jul 29, 2020 · 93ce96d · 93ce96d
1 parent cc2f04b
commit 93ce96d
Show file tree

Hide file tree

Showing 38 changed files with 3,074 additions and 159 deletions.
diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml
@@ -61,6 +61,7 @@ jobs:
           go test -v -timeout 1m ./common -covermode=atomic
           go test -v -timeout 5m ./httpd -covermode=atomic
           go test -v -timeout 5m ./sftpd -covermode=atomic
+          go test -v -timeout 5m ./ftpd -covermode=atomic
         env:
           SFTPGO_DATA_PROVIDER__DRIVER: bolt
           SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
@@ -177,4 +178,4 @@ jobs:
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v1
         with:
-          version: v1.27
+          version: v1.29
diff --git a/README.md b/README.md
@@ -33,9 +33,10 @@ Fully featured and highly configurable SFTP server, written in Go
 - Atomic uploads are configurable.
 - Support for Git repositories over SSH.
 - SCP and rsync are supported.
-- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP.
+- FTP/S is supported.
+- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP/FTP.
 - [Prometheus metrics](./docs/metrics.md) are exposed.
-- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP service without losing the information about the client's address.
+- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP/FTP service without losing the information about the client's address.
 - [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts.
@@ -51,7 +52,8 @@ SFTPGo is developed and tested on Linux. After each commit, the code is automati
 ## Requirements
 
 - Go 1.13 or higher as build only dependency.
-- A suitable SQL server or key/value store to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x
+- A suitable SQL server to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x.
+- The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider.
 
 ## Installation
 

diff --git a/cmd/portable.go b/cmd/portable.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/service"
 	"github.com/drakkan/sftpgo/sftpd"
@@ -49,6 +50,9 @@ var (
 	portableGCSAutoCredentials   int
 	portableGCSStorageClass      string
 	portableGCSKeyPrefix         string
+	portableFTPDPort             int
+	portableFTPSCert             string
+	portableFTPSKey              string
 	portableCmd                  = &cobra.Command{
 		Use:   "portable",
 		Short: "Serve a single directory",
@@ -88,6 +92,14 @@ Please take a look at the usage below to customize the serving parameters`,
 				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
 				portableGCSAutoCredentials = 0
 			}
+			if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 {
+				_, err := common.NewCertManager(portableFTPSCert, portableFTPSKey, "FTP portable")
+				if err != nil {
+					fmt.Printf("Unable to load FTPS key pair, cert file %#v key file %#v error: %v\n",
+						portableFTPSCert, portableFTPSKey, err)
+					os.Exit(1)
+				}
+			}
 			service := service.Service{
 				ConfigDir:     filepath.Clean(defaultConfigDir),
 				ConfigFile:    defaultConfigName,
@@ -133,10 +145,12 @@ Please take a look at the usage below to customize the serving parameters`,
 					},
 				},
 			}
-			if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
-				portableAdvertiseCredentials); err == nil {
+			if err := service.StartPortableMode(portableSFTPDPort, portableFTPDPort, portableSSHCommands, portableAdvertiseService,
+				portableAdvertiseCredentials, portableFTPSCert, portableFTPSKey); err == nil {
 				service.Wait()
-				os.Exit(0)
+				if service.Error == nil {
+					os.Exit(0)
+				}
 			}
 			os.Exit(1)
 		},
@@ -150,7 +164,9 @@ func init() {
 This can be an absolute path or a path
 relative to the current directory
 `)
-	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random  unprivileged port")
+	portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random unprivileged port")
+	portableCmd.Flags().IntVar(&portableFTPDPort, "ftpd-port", -1, `0 means a random unprivileged port,
+< 0 disabled`)
 	portableCmd.Flags().StringSliceVarP(&portableSSHCommands, "ssh-commands", "c", sftpd.GetDefaultSSHCommands(),
 		`SSH commands to enable.
 "*" means any supported SSH command
@@ -177,13 +193,13 @@ insensitive. The format is
 /dir::ext1,ext2.
 For example: "/somedir::.jpg,.png"`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseService, "advertise-service", "S", false,
-		`Advertise SFTP service using multicast
-DNS`)
+		`Advertise SFTP/FTP service using
+multicast DNS`)
 	portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
-		`If the SFTP service is advertised via
-multicast DNS, this flag allows to put
-username/password inside the advertised
-TXT record`)
+		`If the SFTP/FTP service is
+advertised via multicast DNS, this
+flag allows to put username/password
+inside the advertised TXT record`)
 	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, `0 means local filesystem,
 1 Amazon S3 compatible,
 2 Google Cloud Storage`)
@@ -210,6 +226,8 @@ file`)
 	portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, `0 means explicit credentials using
 a JSON credentials file, 1 automatic
 `)
+	portableCmd.Flags().StringVar(&portableFTPSCert, "ftpd-cert", "", "Path to the certificate file for FTPS")
+	portableCmd.Flags().StringVar(&portableFTPSKey, "ftpd-key", "", "Path to the key file for FTPS")
 	rootCmd.AddCommand(portableCmd)
 }
 

diff --git a/common/actions_test.go b/common/actions_test.go
@@ -167,7 +167,7 @@ func TestPreDeleteAction(t *testing.T) {
 	c := NewBaseConnection("id", ProtocolSFTP, user, fs)
 
 	testfile := filepath.Join(user.HomeDir, "testfile")
-	err = ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm)
 	assert.NoError(t, err)
 	info, err := os.Stat(testfile)
 	assert.NoError(t, err)

diff --git a/common/common.go b/common/common.go
@@ -18,6 +18,7 @@ import (
 
 // constants
 const (
+	logSender                = "common"
 	uploadLogSender          = "Upload"
 	downloadLogSender        = "Download"
 	renameLogSender          = "Rename"
@@ -35,7 +36,7 @@ const (
 	operationRename          = "rename"
 	operationSSHCmd          = "ssh_cmd"
 	chtimesFormat            = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS
-	idleTimeoutCheckInterval = 5 * time.Minute
+	idleTimeoutCheckInterval = 3 * time.Minute
 )
 
 // Stat flags
@@ -56,6 +57,7 @@ const (
 	ProtocolSFTP = "SFTP"
 	ProtocolSCP  = "SCP"
 	ProtocolSSH  = "SSH"
+	ProtocolFTP  = "FTP"
 )
 
 // Upload modes
@@ -84,12 +86,13 @@ var (
 	QuotaScans            ActiveScans
 	idleTimeoutTicker     *time.Ticker
 	idleTimeoutTickerDone chan bool
-	supportedProcols      = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH}
+	supportedProcols      = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP}
 )
 
 // Initialize sets the common configuration
 func Initialize(c Configuration) {
 	Config = c
+	Config.idleLoginTimeout = 2 * time.Minute
 	Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
 	if Config.IdleTimeout > 0 {
 		startIdleTimeoutTicker(idleTimeoutCheckInterval)
@@ -220,6 +223,7 @@ type Configuration struct {
 	// connection will be rejected.
 	ProxyAllowed          []string `json:"proxy_allowed" mapstructure:"proxy_allowed"`
 	idleTimeoutAsDuration time.Duration
+	idleLoginTimeout      time.Duration
 }
 
 // IsAtomicUploadEnabled returns true if atomic upload is enabled
@@ -291,35 +295,56 @@ func (conns *ActiveConnections) Add(c ActiveConnection) {
 	logger.Debug(c.GetProtocol(), c.GetID(), "connection added, num open connections: %v", len(conns.connections))
 }
 
+// Swap replaces an existing connection with the given one.
+// This method is useful if you have to change some connection details
+// for example for FTP is used to update the connection once the user
+// authenticates
+func (conns *ActiveConnections) Swap(c ActiveConnection) error {
+	conns.Lock()
+	defer conns.Unlock()
+
+	for idx, conn := range conns.connections {
+		if conn.GetID() == c.GetID() {
+			conn = nil
+			conns.connections[idx] = c
+			return nil
+		}
+	}
+	return errors.New("connection to swap not found")
+}
+
 // Remove removes a connection from the active ones
-func (conns *ActiveConnections) Remove(c ActiveConnection) {
+func (conns *ActiveConnections) Remove(connectionID string) {
 	conns.Lock()
 	defer conns.Unlock()
 
+	var c ActiveConnection
 	indexToRemove := -1
-	for i, v := range conns.connections {
-		if v.GetID() == c.GetID() {
+	for i, conn := range conns.connections {
+		if conn.GetID() == connectionID {
 			indexToRemove = i
+			c = conn
 			break
 		}
 	}
 	if indexToRemove >= 0 {
 		conns.connections[indexToRemove] = conns.connections[len(conns.connections)-1]
 		conns.connections[len(conns.connections)-1] = nil
 		conns.connections = conns.connections[:len(conns.connections)-1]
+		metrics.UpdateActiveConnectionsSize(len(conns.connections))
 		logger.Debug(c.GetProtocol(), c.GetID(), "connection removed, num open connections: %v",
 			len(conns.connections))
+		// we have finished to send data here and most of the time the underlying network connection
+		// is already closed. Sometime a client can still be reading the last sended data, so we set
+		// a deadline instead of directly closing the network connection.
+		// Setting a deadline on an already closed connection has no effect.
+		// We only need to ensure that a connection will not remain indefinitely open and so the
+		// underlying file descriptor is not released.
+		// This should protect us against buggy clients and edge cases.
+		c.SetConnDeadline()
 	} else {
-		logger.Warn(c.GetProtocol(), c.GetID(), "connection to remove not found!")
+		logger.Warn(logSender, "", "connection to remove with id %#v not found!", connectionID)
 	}
-	// we have finished to send data here and most of the time the underlying network connection
-	// is already closed. Sometime a client can still be reading the last sended data, so we set
-	// a deadline instead of directly closing the network connection.
-	// Setting a deadline on an already closed connection has no effect.
-	// We only need to ensure that a connection will not remain indefinitely open and so the
-	// underlying file descriptor is not released.
-	// This should protect us against buggy clients and edge cases.
-	c.SetConnDeadline()
 }
 
 // Close closes an active connection.
@@ -330,10 +355,10 @@ func (conns *ActiveConnections) Close(connectionID string) bool {
 
 	for _, c := range conns.connections {
 		if c.GetID() == connectionID {
-			defer func() {
-				err := c.Disconnect()
-				logger.Debug(c.GetProtocol(), c.GetID(), "close connection requested, close err: %v", err)
-			}()
+			defer func(conn ActiveConnection) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err)
+			}(c)
 			result = true
 			break
 		}
@@ -348,11 +373,19 @@ func (conns *ActiveConnections) checkIdleConnections() {
 
 	for _, c := range conns.connections {
 		idleTime := time.Since(c.GetLastActivity())
-		if idleTime > Config.idleTimeoutAsDuration {
-			defer func() {
-				err := c.Disconnect()
-				logger.Debug(c.GetProtocol(), c.GetID(), "close idle connection, idle time: %v, close err: %v", idleTime, err)
-			}()
+		isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && len(c.GetUsername()) == 0)
+
+		if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) {
+			defer func(conn ActiveConnection, isFTPNoAuth bool) {
+				err := conn.Disconnect()
+				logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %v, username: %#v close err: %v",
+					idleTime, conn.GetUsername(), err)
+				if isFTPNoAuth {
+					logger.ConnectionFailedLog("", utils.GetIPFromRemoteAddress(c.GetRemoteAddress()),
+						"no_auth_tryed", "client idle")
+					metrics.AddNoAuthTryed()
+				}
+			}(c, isUnauthenticatedFTPUser)
 		}
 	}
 

diff --git a/common/common_test.go b/common/common_test.go
@@ -19,7 +19,7 @@ import (
 )
 
 const (
-	logSender        = "common_test"
+	logSenderTest    = "common_test"
 	httpAddr         = "127.0.0.1:9999"
 	httpProxyAddr    = "127.0.0.1:7777"
 	configDir        = ".."
@@ -37,8 +37,18 @@ type fakeConnection struct {
 	sshCommand string
 }
 
+func (c *fakeConnection) AddUser(user dataprovider.User) error {
+	fs, err := user.GetFilesystem(c.GetID())
+	if err != nil {
+		return err
+	}
+	c.BaseConnection.User = user
+	c.BaseConnection.Fs = fs
+	return nil
+}
+
 func (c *fakeConnection) Disconnect() error {
-	Connections.Remove(c)
+	Connections.Remove(c.GetID())
 	return nil
 }
 
@@ -166,16 +176,30 @@ func TestIdleConnections(t *testing.T) {
 	user := dataprovider.User{
 		Username: username,
 	}
-	c := NewBaseConnection("id", ProtocolSFTP, user, nil)
+	c := NewBaseConnection("id1", ProtocolSFTP, user, nil)
 	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
 	fakeConn := &fakeConnection{
 		BaseConnection: c,
 	}
 	Connections.Add(fakeConn)
 	assert.Equal(t, Connections.GetActiveSessions(username), 1)
+	c = NewBaseConnection("id2", ProtocolFTP, dataprovider.User{}, nil)
+	c.lastActivity = time.Now().UnixNano()
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	assert.Equal(t, Connections.GetActiveSessions(username), 1)
+	assert.Len(t, Connections.GetStats(), 2)
+
 	startIdleTimeoutTicker(100 * time.Millisecond)
 	assert.Eventually(t, func() bool { return Connections.GetActiveSessions(username) == 0 }, 1*time.Second, 200*time.Millisecond)
 	stopIdleTimeoutTicker()
+	assert.Len(t, Connections.GetStats(), 1)
+	c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano()
+	startIdleTimeoutTicker(100 * time.Millisecond)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 1*time.Second, 200*time.Millisecond)
+	stopIdleTimeoutTicker()
 
 	Config = configCopy
 }
@@ -192,7 +216,34 @@ func TestCloseConnection(t *testing.T) {
 	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
 	res = Connections.Close(fakeConn.GetID())
 	assert.False(t, res)
-	Connections.Remove(fakeConn)
+	Connections.Remove(fakeConn.GetID())
+}
+
+func TestSwapConnection(t *testing.T) {
+	c := NewBaseConnection("id", ProtocolFTP, dataprovider.User{}, nil)
+	fakeConn := &fakeConnection{
+		BaseConnection: c,
+	}
+	Connections.Add(fakeConn)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, "", Connections.GetStats()[0].Username)
+	}
+	c = NewBaseConnection("id", ProtocolFTP, dataprovider.User{
+		Username: userTestUsername,
+	}, nil)
+	fakeConn = &fakeConnection{
+		BaseConnection: c,
+	}
+	err := Connections.Swap(fakeConn)
+	assert.NoError(t, err)
+	if assert.Len(t, Connections.GetStats(), 1) {
+		assert.Equal(t, userTestUsername, Connections.GetStats()[0].Username)
+	}
+	res := Connections.Close(fakeConn.GetID())
+	assert.True(t, res)
+	assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond)
+	err = Connections.Swap(fakeConn)
+	assert.Error(t, err)
 }
 
 func TestAtomicUpload(t *testing.T) {
@@ -255,8 +306,8 @@ func TestConnectionStatus(t *testing.T) {
 	err = t2.Close()
 	assert.NoError(t, err)
 
-	Connections.Remove(fakeConn1)
-	Connections.Remove(fakeConn2)
+	Connections.Remove(fakeConn1.GetID())
+	Connections.Remove(fakeConn2.GetID())
 	stats = Connections.GetStats()
 	assert.Len(t, stats, 0)
 }