Skip to content

Commit

Permalink
add KMS support
Browse files Browse the repository at this point in the history 
Fixes drakkan#226
  • Loading branch information
@drakkan
drakkan committed Nov 30, 2020
1 parent af0c9b7 commit 634b723
Show file tree
Hide file tree
Showing 46 changed files with 1,582 additions and 536 deletions.
28 changes: 10 additions & 18 deletions cmd/portable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/drakkan/sftpgo/common"
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/service"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/version"
Expand Down Expand Up @@ -143,36 +144,27 @@ Please take a look at the usage below to customize the serving parameters`,
FsConfig: dataprovider.Filesystem{
Provider: dataprovider.FilesystemProvider(portableFsProvider),
S3Config: vfs.S3FsConfig{
Bucket: portableS3Bucket,
Region: portableS3Region,
AccessKey: portableS3AccessKey,
AccessSecret: vfs.Secret{
Status: vfs.SecretStatusPlain,
Payload: portableS3AccessSecret,
},
Bucket: portableS3Bucket,
Region: portableS3Region,
AccessKey: portableS3AccessKey,
AccessSecret: kms.NewPlainSecret(portableS3AccessSecret),
Endpoint: portableS3Endpoint,
StorageClass: portableS3StorageClass,
KeyPrefix: portableS3KeyPrefix,
UploadPartSize: int64(portableS3ULPartSize),
UploadConcurrency: portableS3ULConcurrency,
},
GCSConfig: vfs.GCSFsConfig{
Bucket: portableGCSBucket,
Credentials: vfs.Secret{
Status: vfs.SecretStatusPlain,
Payload: string(portableGCSCredentials),
},
Bucket: portableGCSBucket,
Credentials: kms.NewPlainSecret(string(portableGCSCredentials)),
AutomaticCredentials: portableGCSAutoCredentials,
StorageClass: portableGCSStorageClass,
KeyPrefix: portableGCSKeyPrefix,
},
AzBlobConfig: vfs.AzBlobFsConfig{
Container: portableAzContainer,
AccountName: portableAzAccountName,
AccountKey: vfs.Secret{
Status: vfs.SecretStatusPlain,
Payload: portableAzAccountKey,
},
Container: portableAzContainer,
AccountName: portableAzAccountName,
AccountKey: kms.NewPlainSecret(portableAzAccountKey),
Endpoint: portableAzEndpoint,
AccessTier: portableAzAccessTier,
SASURL: portableAzSASURL,
Expand Down
5 changes: 5 additions & 0 deletions cmd/startsubsys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ Command-line flags should be specified in the Subsystem declaration.
}
httpConfig := config.GetHTTPConfig()
httpConfig.Initialize(configDir)
kmsConfig := config.GetKMSConfig()
if err := kmsConfig.Initialize(); err != nil {
logger.Error(logSender, connectionID, "unable to initialize KMS: %v", err)
os.Exit(1)
}
user, err := dataprovider.UserExists(username)
if err == nil {
if user.HomeDir != filepath.Clean(homedir) && !preserveHomeDir {
Expand Down
20 changes: 20 additions & 0 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/drakkan/sftpgo/ftpd"
"github.com/drakkan/sftpgo/httpclient"
"github.com/drakkan/sftpgo/httpd"
"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/utils"
Expand Down Expand Up @@ -43,6 +44,7 @@ type globalConfig struct {
ProviderConf dataprovider.Config `json:"data_provider" mapstructure:"data_provider"`
HTTPDConfig httpd.Conf `json:"httpd" mapstructure:"httpd"`
HTTPConfig httpclient.Config `json:"http" mapstructure:"http"`
KMSConfig kms.Configuration `json:"kms" mapstructure:"kms"`
}

func init() {
Expand Down Expand Up @@ -164,6 +166,12 @@ func init() {
CACertificates: nil,
SkipTLSVerify: false,
},
KMSConfig: kms.Configuration{
Secrets: kms.Secrets{
URL: "",
MasterKeyPath: "",
},
},
}

viper.SetEnvPrefix(configEnvPrefix)
Expand Down Expand Up @@ -240,6 +248,16 @@ func GetHTTPConfig() httpclient.Config {
return globalConf.HTTPConfig
}

// GetKMSConfig returns the KMS configuration
func GetKMSConfig() kms.Configuration {
return globalConf.KMSConfig
}

// SetKMSConfig sets the kms configuration
func SetKMSConfig(config kms.Configuration) {
globalConf.KMSConfig = config
}

// HasServicesToStart returns true if the config defines at least a service to start.
// Supported services are SFTP, FTP and WebDAV
func HasServicesToStart() bool {
Expand Down Expand Up @@ -456,4 +474,6 @@ func setViperDefaults() {
viper.SetDefault("http.timeout", globalConf.HTTPConfig.Timeout)
viper.SetDefault("http.ca_certificates", globalConf.HTTPConfig.CACertificates)
viper.SetDefault("http.skip_tls_verify", globalConf.HTTPConfig.SkipTLSVerify)
viper.SetDefault("kms.secrets.url", globalConf.KMSConfig.Secrets.URL)
viper.SetDefault("kms.secrets.master_key_path", globalConf.KMSConfig.Secrets.MasterKeyPath)
}
13 changes: 13 additions & 0 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,12 @@ func TestSetGetConfig(t *testing.T) {
config.SetWebDAVDConfig(webDavConf)
assert.Equal(t, webDavConf.CertificateFile, config.GetWebDAVDConfig().CertificateFile)
assert.Equal(t, webDavConf.CertificateKeyFile, config.GetWebDAVDConfig().CertificateKeyFile)
kmsConf := config.GetKMSConfig()
kmsConf.Secrets.MasterKeyPath = "apath"
kmsConf.Secrets.URL = "aurl"
config.SetKMSConfig(kmsConf)
assert.Equal(t, kmsConf.Secrets.MasterKeyPath, config.GetKMSConfig().Secrets.MasterKeyPath)
assert.Equal(t, kmsConf.Secrets.URL, config.GetKMSConfig().Secrets.URL)
}

func TestServiceToStart(t *testing.T) {
Expand Down Expand Up @@ -313,11 +319,15 @@ func TestConfigFromEnv(t *testing.T) {
os.Setenv("SFTPGO_DATA_PROVIDER__PASSWORD_HASHING__ARGON2_OPTIONS__ITERATIONS", "41")
os.Setenv("SFTPGO_DATA_PROVIDER__POOL_SIZE", "10")
os.Setenv("SFTPGO_DATA_PROVIDER__ACTIONS__EXECUTE_ON", "add")
os.Setenv("SFTPGO_KMS__SECRETS__URL", "local")
os.Setenv("SFTPGO_KMS__SECRETS__MASTER_KEY_PATH", "path")
t.Cleanup(func() {
os.Unsetenv("SFTPGO_SFTPD__BIND_ADDRESS")
os.Unsetenv("SFTPGO_DATA_PROVIDER__PASSWORD_HASHING__ARGON2_OPTIONS__ITERATIONS")
os.Unsetenv("SFTPGO_DATA_PROVIDER__POOL_SIZE")
os.Unsetenv("SFTPGO_DATA_PROVIDER__ACTIONS__EXECUTE_ON")
os.Unsetenv("SFTPGO_KMS__SECRETS__URL")
os.Unsetenv("SFTPGO_KMS__SECRETS__MASTER_KEY_PATH")
})
err := config.LoadConfig(".", "invalid config")
assert.NoError(t, err)
Expand All @@ -328,4 +338,7 @@ func TestConfigFromEnv(t *testing.T) {
assert.Equal(t, 10, dataProviderConf.PoolSize)
assert.Len(t, dataProviderConf.Actions.ExecuteOn, 1)
assert.Contains(t, dataProviderConf.Actions.ExecuteOn, "add")
kmsConfig := config.GetKMSConfig()
assert.Equal(t, "local", kmsConfig.Secrets.URL)
assert.Equal(t, "path", kmsConfig.Secrets.MasterKeyPath)
}
1 change: 1 addition & 0 deletions dataprovider/bolt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,7 @@ func joinUserAndFolders(u []byte, foldersBucket *bolt.Bucket) (User, error) {
}
user.VirtualFolders = folders
}
user.SetEmptySecretsIfNil()
return user, err
}

Expand Down
29 changes: 16 additions & 13 deletions dataprovider/compat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io/ioutil"
"path/filepath"

"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
Expand Down Expand Up @@ -129,6 +130,7 @@ func createUserFromV4(u compatUserV4, fsConfig Filesystem) User {
Filters: u.Filters,
}
user.FsConfig = fsConfig
user.SetEmptySecretsIfNil()
return user
}

Expand Down Expand Up @@ -160,27 +162,28 @@ func convertUserToV4(u User, fsConfig compatFilesystemV4) compatUserV4 {
return user
}

func getCGSCredentialsFromV4(config compatGCSFsConfigV4) (vfs.Secret, error) {
var secret vfs.Secret
func getCGSCredentialsFromV4(config compatGCSFsConfigV4) (*kms.Secret, error) {
secret := kms.NewEmptySecret()
var err error
if len(config.Credentials) > 0 {
secret.Status = vfs.SecretStatusPlain
secret.Payload = string(config.Credentials)
secret = kms.NewPlainSecret(string(config.Credentials))
return secret, nil
}
if config.CredentialFile != "" {
creds, err := ioutil.ReadFile(config.CredentialFile)
if err != nil {
return secret, err
}
secret.Status = vfs.SecretStatusPlain
secret.Payload = string(creds)
secret = kms.NewPlainSecret(string(creds))
return secret, nil
}
return secret, err
}

func getCGSCredentialsFromV6(config vfs.GCSFsConfig, username string) (string, error) {
if config.Credentials == nil {
config.Credentials = kms.NewEmptySecret()
}
if config.Credentials.IsEmpty() {
config.CredentialFile = filepath.Join(credentialsDirPath, fmt.Sprintf("%v_gcs_credentials.json",
username))
Expand All @@ -199,7 +202,7 @@ func getCGSCredentialsFromV6(config vfs.GCSFsConfig, username string) (string, e
return "", err
}
// in V4 GCS credentials were not encrypted
return config.Credentials.Payload, nil
return config.Credentials.GetPayload(), nil
}
return "", nil
}
Expand Down Expand Up @@ -229,7 +232,7 @@ func convertFsConfigToV4(fs Filesystem, username string) (compatFilesystemV4, er
if err != nil {
return fsV4, err
}
secretV4, err := utils.EncryptData(fs.S3Config.AccessSecret.Payload)
secretV4, err := utils.EncryptData(fs.S3Config.AccessSecret.GetPayload())
if err != nil {
return fsV4, err
}
Expand All @@ -253,7 +256,7 @@ func convertFsConfigToV4(fs Filesystem, username string) (compatFilesystemV4, er
if err != nil {
return fsV4, err
}
secretV4, err := utils.EncryptData(fs.AzBlobConfig.AccountKey.Payload)
secretV4, err := utils.EncryptData(fs.AzBlobConfig.AccountKey.GetPayload())
if err != nil {
return fsV4, err
}
Expand Down Expand Up @@ -292,14 +295,14 @@ func convertFsConfigFromV4(compatFs compatFilesystemV4, username string) (Filesy
KeyPrefix: compatFs.S3Config.KeyPrefix,
Region: compatFs.S3Config.Region,
AccessKey: compatFs.S3Config.AccessKey,
AccessSecret: vfs.Secret{},
AccessSecret: kms.NewEmptySecret(),
Endpoint: compatFs.S3Config.Endpoint,
StorageClass: compatFs.S3Config.StorageClass,
UploadPartSize: compatFs.S3Config.UploadPartSize,
UploadConcurrency: compatFs.S3Config.UploadConcurrency,
}
if compatFs.S3Config.AccessSecret != "" {
secret, err := vfs.GetSecretFromCompatString(compatFs.S3Config.AccessSecret)
secret, err := kms.GetSecretFromCompatString(compatFs.S3Config.AccessSecret)
if err != nil {
providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err)
return fsConfig, err
Expand All @@ -310,7 +313,7 @@ func convertFsConfigFromV4(compatFs compatFilesystemV4, username string) (Filesy
fsConfig.AzBlobConfig = vfs.AzBlobFsConfig{
Container: compatFs.AzBlobConfig.Container,
AccountName: compatFs.AzBlobConfig.AccountName,
AccountKey: vfs.Secret{},
AccountKey: kms.NewEmptySecret(),
Endpoint: compatFs.AzBlobConfig.Endpoint,
SASURL: compatFs.AzBlobConfig.SASURL,
KeyPrefix: compatFs.AzBlobConfig.KeyPrefix,
Expand All @@ -320,7 +323,7 @@ func convertFsConfigFromV4(compatFs compatFilesystemV4, username string) (Filesy
AccessTier: compatFs.AzBlobConfig.AccessTier,
}
if compatFs.AzBlobConfig.AccountKey != "" {
secret, err := vfs.GetSecretFromCompatString(compatFs.AzBlobConfig.AccountKey)
secret, err := kms.GetSecretFromCompatString(compatFs.AzBlobConfig.AccountKey)
if err != nil {
providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err)
return fsConfig, err
Expand Down
26 changes: 17 additions & 9 deletions dataprovider/dataprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"golang.org/x/crypto/ssh"

"github.com/drakkan/sftpgo/httpclient"
"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/metrics"
"github.com/drakkan/sftpgo/utils"
Expand Down Expand Up @@ -124,6 +125,7 @@ var (
sqlTableFoldersMapping = "folders_mapping"
sqlTableSchemaVersion = "schema_version"
argon2Params *argon2id.Params
lastLoginMinDelay = 10 * time.Minute
)

type schemaVersion struct {
Expand Down Expand Up @@ -577,7 +579,12 @@ func UpdateLastLogin(user User) error {
if config.ManageUsers == 0 {
return &MethodDisabledError{err: manageUsersDisabledError}
}
return provider.updateLastLogin(user.Username)
lastLogin := utils.GetTimeFromMsecSinceEpoch(user.LastLogin)
diff := -time.Until(lastLogin)
if diff < 0 || diff > lastLoginMinDelay {
return provider.updateLastLogin(user.Username)
}
return nil
}

// UpdateUserQuota updates the quota for the given SFTP user adding filesAdd and sizeAdd.
Expand Down Expand Up @@ -1099,12 +1106,12 @@ func saveGCSCredentials(user *User) error {
if user.FsConfig.Provider != GCSFilesystemProvider {
return nil
}
if user.FsConfig.GCSConfig.Credentials.Payload == "" {
if user.FsConfig.GCSConfig.Credentials.GetPayload() == "" {
return nil
}
if config.PreferDatabaseCredentials {
if user.FsConfig.GCSConfig.Credentials.IsPlain() {
user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username
user.FsConfig.GCSConfig.Credentials.SetAdditionalData(user.Username)
err := user.FsConfig.GCSConfig.Credentials.Encrypt()
if err != nil {
return err
Expand All @@ -1113,7 +1120,7 @@ func saveGCSCredentials(user *User) error {
return nil
}
if user.FsConfig.GCSConfig.Credentials.IsPlain() {
user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username
user.FsConfig.GCSConfig.Credentials.SetAdditionalData(user.Username)
err := user.FsConfig.GCSConfig.Credentials.Encrypt()
if err != nil {
return &ValidationError{err: fmt.Sprintf("could not encrypt GCS credentials: %v", err)}
Expand All @@ -1132,7 +1139,7 @@ func saveGCSCredentials(user *User) error {
if err != nil {
return &ValidationError{err: fmt.Sprintf("could not save GCS credentials: %v", err)}
}
user.FsConfig.GCSConfig.Credentials = vfs.Secret{}
user.FsConfig.GCSConfig.Credentials = kms.NewEmptySecret()
return nil
}

Expand All @@ -1143,7 +1150,7 @@ func validateFilesystemConfig(user *User) error {
return &ValidationError{err: fmt.Sprintf("could not validate s3config: %v", err)}
}
if user.FsConfig.S3Config.AccessSecret.IsPlain() {
user.FsConfig.S3Config.AccessSecret.AdditionalData = user.Username
user.FsConfig.S3Config.AccessSecret.SetAdditionalData(user.Username)
err = user.FsConfig.S3Config.AccessSecret.Encrypt()
if err != nil {
return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)}
Expand All @@ -1166,7 +1173,7 @@ func validateFilesystemConfig(user *User) error {
return &ValidationError{err: fmt.Sprintf("could not validate Azure Blob config: %v", err)}
}
if user.FsConfig.AzBlobConfig.AccountKey.IsPlain() {
user.FsConfig.AzBlobConfig.AccountKey.AdditionalData = user.Username
user.FsConfig.AzBlobConfig.AccountKey.SetAdditionalData(user.Username)
err = user.FsConfig.AzBlobConfig.AccountKey.Encrypt()
if err != nil {
return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)}
Expand Down Expand Up @@ -1220,6 +1227,7 @@ func validateFolder(folder *vfs.BaseVirtualFolder) error {
}

func validateUser(user *User) error {
user.SetEmptySecretsIfNil()
buildUserHomeDir(user)
if err := validateBaseParams(user); err != nil {
return err
Expand Down Expand Up @@ -2131,7 +2139,7 @@ func CacheWebDAVUser(cachedUser *CachedUser, maxSize int) {
}
}

if len(cachedUser.User.Username) > 0 {
if cachedUser.User.Username != "" {
webDAVUsersCache.Store(cachedUser.User.Username, cachedUser)
}
}
Expand All @@ -2143,7 +2151,7 @@ func GetCachedWebDAVUser(username string) (interface{}, bool) {

// RemoveCachedWebDAVUser removes a cached WebDAV user
func RemoveCachedWebDAVUser(username string) {
if len(username) > 0 {
if username != "" {
webDAVUsersCache.Delete(username)
}
}
Loading

0 comments on commit 634b723

Please sign in to comment.