Skip to content

Commit

Permalink
telemetry server: add optional https and authentication
Browse files Browse the repository at this point in the history
  • Loading branch information
drakkan committed Dec 18, 2020
1 parent 1403807 commit bcf0fa0
Show file tree
Hide file tree
Showing 21 changed files with 491 additions and 168 deletions.
1 change: 1 addition & 0 deletions .github/workflows/development.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ jobs:
go test -v -p 1 -timeout 8m ./sftpd -covermode=atomic
go test -v -p 1 -timeout 2m ./ftpd -covermode=atomic
go test -v -p 1 -timeout 2m ./webdavd -covermode=atomic
go test -v -p 1 -timeout 2m ./telemetry -covermode=atomic
env:
SFTPGO_DATA_PROVIDER__DRIVER: bolt
SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db'
Expand Down
1 change: 0 additions & 1 deletion cmd/portable.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ Please take a look at the usage below to customize the serving parameters`,
LogMaxAge: defaultLogMaxAge,
LogCompress: defaultLogCompress,
LogVerbose: portableLogVerbose,
Profiler: defaultProfiler,
Shutdown: make(chan bool),
PortableMode: 1,
PortableUser: dataprovider.User{
Expand Down
14 changes: 0 additions & 14 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ const (
logCompressKey = "log_compress"
logVerboseFlag = "log-verbose"
logVerboseKey = "log_verbose"
profilerFlag = "profiler"
profilerKey = "profiler"
loadDataFromFlag = "loaddata-from"
loadDataFromKey = "loaddata_from"
loadDataModeFlag = "loaddata-mode"
Expand All @@ -46,7 +44,6 @@ const (
defaultLogMaxAge = 28
defaultLogCompress = false
defaultLogVerbose = true
defaultProfiler = false
defaultLoadDataFrom = ""
defaultLoadDataMode = 1
defaultLoadDataQuotaScan = 0
Expand All @@ -62,7 +59,6 @@ var (
logMaxAge int
logCompress bool
logVerbose bool
profiler bool
loadDataFrom string
loadDataMode int
loadDataQuotaScan int
Expand Down Expand Up @@ -183,16 +179,6 @@ using SFTPGO_LOG_VERBOSE env var too.
`)
viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck

viper.SetDefault(profilerKey, defaultProfiler)
viper.BindEnv(profilerKey, "SFTPGO_PROFILER") //nolint:errcheck
cmd.Flags().BoolVarP(&profiler, profilerFlag, "p", viper.GetBool(profilerKey),
`Enable the built-in profiler. The profiler will
be accessible via HTTP/HTTPS using the base URL
"/debug/pprof/".
This flag can be set using SFTPGO_PROFILER env
var too.`)
viper.BindPFlag(profilerKey, cmd.Flags().Lookup(profilerFlag)) //nolint:errcheck

viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
Expand Down
1 change: 0 additions & 1 deletion cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ Please take a look at the usage below to customize the startup options`,
LoadDataMode: loadDataMode,
LoadDataQuotaScan: loadDataQuotaScan,
LoadDataClean: loadDataClean,
Profiler: profiler,
Shutdown: make(chan bool),
}
if err := service.Start(); err == nil {
Expand Down
134 changes: 134 additions & 0 deletions common/httpauth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package common

import (
"encoding/csv"
"os"
"strings"
"sync"

"github.com/GehirnInc/crypt/apr1_crypt"
"github.com/GehirnInc/crypt/md5_crypt"
"golang.org/x/crypto/bcrypt"

"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
)

const (
// HTTPAuthenticationHeader defines the HTTP authentication
HTTPAuthenticationHeader = "WWW-Authenticate"
md5CryptPwdPrefix = "$1$"
apr1CryptPwdPrefix = "$apr1$"
)

var (
bcryptPwdPrefixes = []string{"$2a$", "$2$", "$2x$", "$2y$", "$2b$"}
)

// HTTPAuthProvider defines the interface for HTTP auth providers
type HTTPAuthProvider interface {
ValidateCredentials(username, password string) bool
IsEnabled() bool
}

type basicAuthProvider struct {
Path string
sync.RWMutex
Info os.FileInfo
Users map[string]string
}

// NewBasicAuthProvider returns an HTTPAuthProvider implementing Basic Auth
func NewBasicAuthProvider(authUserFile string) (HTTPAuthProvider, error) {
basicAuthProvider := basicAuthProvider{
Path: authUserFile,
Info: nil,
Users: make(map[string]string),
}
return &basicAuthProvider, basicAuthProvider.loadUsers()
}

func (p *basicAuthProvider) IsEnabled() bool {
return p.Path != ""
}

func (p *basicAuthProvider) isReloadNeeded(info os.FileInfo) bool {
p.RLock()
defer p.RUnlock()

return p.Info == nil || p.Info.ModTime() != info.ModTime() || p.Info.Size() != info.Size()
}

func (p *basicAuthProvider) loadUsers() error {
if !p.IsEnabled() {
return nil
}
info, err := os.Stat(p.Path)
if err != nil {
logger.Debug(logSender, "", "unable to stat basic auth users file: %v", err)
return err
}
if p.isReloadNeeded(info) {
r, err := os.Open(p.Path)
if err != nil {
logger.Debug(logSender, "", "unable to open basic auth users file: %v", err)
return err
}
defer r.Close()
reader := csv.NewReader(r)
reader.Comma = ':'
reader.Comment = '#'
reader.TrimLeadingSpace = true
records, err := reader.ReadAll()
if err != nil {
logger.Debug(logSender, "", "unable to parse basic auth users file: %v", err)
return err
}
p.Lock()
defer p.Unlock()

p.Users = make(map[string]string)
for _, record := range records {
if len(record) == 2 {
p.Users[record[0]] = record[1]
}
}
logger.Debug(logSender, "", "number of users loaded for httpd basic auth: %v", len(p.Users))
p.Info = info
}
return nil
}

func (p *basicAuthProvider) getHashedPassword(username string) (string, bool) {
err := p.loadUsers()
if err != nil {
return "", false
}
p.RLock()
defer p.RUnlock()

pwd, ok := p.Users[username]
return pwd, ok
}

// ValidateCredentials returns true if the credentials are valid
func (p *basicAuthProvider) ValidateCredentials(username, password string) bool {
if hashedPwd, ok := p.getHashedPassword(username); ok {
if utils.IsStringPrefixInSlice(hashedPwd, bcryptPwdPrefixes) {
err := bcrypt.CompareHashAndPassword([]byte(hashedPwd), []byte(password))
return err == nil
}
if strings.HasPrefix(hashedPwd, md5CryptPwdPrefix) {
crypter := md5_crypt.New()
err := crypter.Verify(hashedPwd, []byte(password))
return err == nil
}
if strings.HasPrefix(hashedPwd, apr1CryptPwdPrefix) {
crypter := apr1_crypt.New()
err := crypter.Verify(hashedPwd, []byte(password))
return err == nil
}
}

return false
}
72 changes: 72 additions & 0 deletions common/httpauth_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package common

import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"

"github.com/stretchr/testify/require"
)

func TestBasicAuth(t *testing.T) {
httpAuth, err := NewBasicAuthProvider("")
require.NoError(t, err)
require.False(t, httpAuth.IsEnabled())

_, err = NewBasicAuthProvider("missing path")
require.Error(t, err)

authUserFile := filepath.Join(os.TempDir(), "http_users.txt")
authUserData := []byte("test1:$2y$05$bcHSED7aO1cfLto6ZdDBOOKzlwftslVhtpIkRhAtSa4GuLmk5mola\n")
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)

httpAuth, err = NewBasicAuthProvider(authUserFile)
require.NoError(t, err)
require.True(t, httpAuth.IsEnabled())
require.False(t, httpAuth.ValidateCredentials("test1", "wrong1"))
require.False(t, httpAuth.ValidateCredentials("test2", "password2"))
require.True(t, httpAuth.ValidateCredentials("test1", "password1"))

authUserData = append(authUserData, []byte("test2:$1$OtSSTL8b$bmaCqEksI1e7rnZSjsIDR1\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test2", "wrong2"))
require.True(t, httpAuth.ValidateCredentials("test2", "password2"))

authUserData = append(authUserData, []byte("test2:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test2", "wrong2"))
require.True(t, httpAuth.ValidateCredentials("test2", "password2"))

authUserData = append(authUserData, []byte("test3:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test3", "password3"))

authUserData = append(authUserData, []byte("test4:$invalid$gLnIkRIf$Xr/6$aJfmIr$ihP4b2N2tcs/\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test4", "password3"))

if runtime.GOOS != "windows" {
authUserData = append(authUserData, []byte("test5:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
err = os.Chmod(authUserFile, 0001)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test5", "password2"))
err = os.Chmod(authUserFile, os.ModePerm)
require.NoError(t, err)
}
authUserData = append(authUserData, []byte("\"foo\"bar\"\r\n")...)
err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm)
require.NoError(t, err)
require.False(t, httpAuth.ValidateCredentials("test2", "password2"))

err = os.Remove(authUserFile)
require.NoError(t, err)
}
12 changes: 10 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,12 @@ func Init() {
},
},
TelemetryConfig: telemetry.Conf{
BindPort: 10000,
BindAddress: "127.0.0.1",
BindPort: 10000,
BindAddress: "127.0.0.1",
EnableProfiler: false,
AuthUserFile: "",
CertificateFile: "",
CertificateKeyFile: "",
},
}

Expand Down Expand Up @@ -514,4 +518,8 @@ func setViperDefaults() {
viper.SetDefault("kms.secrets.master_key_path", globalConf.KMSConfig.Secrets.MasterKeyPath)
viper.SetDefault("telemetry.bind_port", globalConf.TelemetryConfig.BindPort)
viper.SetDefault("telemetry.bind_address", globalConf.TelemetryConfig.BindAddress)
viper.SetDefault("telemetry.enable_profiler", globalConf.TelemetryConfig.EnableProfiler)
viper.SetDefault("telemetry.auth_user_file", globalConf.TelemetryConfig.AuthUserFile)
viper.SetDefault("telemetry.certificate_file", globalConf.TelemetryConfig.CertificateFile)
viper.SetDefault("telemetry.certificate_key_file", globalConf.TelemetryConfig.CertificateKeyFile)
}
4 changes: 4 additions & 0 deletions docs/full-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ The configuration file contains the following sections:
- **"telemetry"**, the configuration for the telemetry server, more details [below](#telemetry-server)
- `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 10000
- `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
- `enable_profiler`, boolean. Enable the built-in profiler. Default `false`
- `auth_user_file`, string. Path to a file used to store usernames and passwords for basic authentication. This can be an absolute path or a path relative to the config dir. We support HTTP basic authentication, and the file format must conform to the one generated using the Apache `htpasswd` tool. The supported password formats are bcrypt (`$2y$` prefix) and md5 crypt (`$apr1$` prefix). If empty, HTTP authentication is disabled. Authentication will be always disabled for the `/healthz` endpoint.
- `certificate_file`, string. Certificate for HTTPS. This can be an absolute path or a path relative to the config dir.
- `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided, the server will expect HTTPS connections. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
- **"http"**, the configuration for HTTP clients. HTTP clients are used for executing hooks such as the ones used for custom actions, external authentication and pre-login user modifications
- `timeout`, integer. Timeout specifies a time limit, in seconds, for requests.
- `ca_certificates`, list of strings. List of paths to extra CA certificates to trust. The paths can be absolute or relative to the config dir. Adding trusted CA certificates is a convenient way to use self-signed certificates without defeating the purpose of using TLS.
Expand Down
4 changes: 3 additions & 1 deletion docs/metrics.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Metrics

SFTPGo exposes [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint.
SFTPGo exposes [Prometheus](https://prometheus.io/) metrics at the `/metrics` HTTP endpoint of the telemetry server.
Several counters and gauges are available, for example:

- Total uploads and downloads
Expand All @@ -16,3 +16,5 @@ Several counters and gauges are available, for example:
- Process information like CPU, memory, file descriptor usage and start time

Please check the `/metrics` page for more details.

We expose the `/metrics` endpoint in both HTTP server and the telemetry server, you should use the one from the telemetry server. The HTTP server `/metrics` endpoint is deprecated and it will be removed in future releases.
2 changes: 1 addition & 1 deletion docs/profiling.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Profiling SFTPGo

The built-in profiler lets you collect CPU profiles, traces, allocations and heap profiles that allow to identify and correct specific bottlenecks.
You can enable the built-in profiler using the `--profiler` command flag.
You can enable the built-in profiler using `telemetry` configuration section inside the configuration file.

Profiling data are exposed via HTTP/HTTPS in the format expected by the [pprof](https://github.com/google/pprof/blob/master/doc/README.md) visualization tool. You can find the index page at the URL `/debug/pprof/`.

Expand Down
4 changes: 2 additions & 2 deletions httpd/api_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ func sendHTTPRequest(method, url string, body io.Reader, contentType string) (*h
if err != nil {
return nil, err
}
if len(contentType) > 0 {
if contentType != "" {
req.Header.Set("Content-Type", "application/json")
}
if len(authUsername) > 0 || len(authPassword) > 0 {
if authUsername != "" || authPassword != "" {
req.SetBasicAuth(authUsername, authPassword)
}
return httpclient.GetHTTPClient().Do(req)
Expand Down
Loading

0 comments on commit bcf0fa0

Please sign in to comment.