initprovider: add load data options

Fixes drakkan#741 Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
devopsmakers · Feb 28, 2022 · 1ea7429 · 1ea7429
1 parent 816c174
commit 1ea7429
Show file tree

Hide file tree

Showing 10 changed files with 84 additions and 63 deletions.
diff --git a/cmd/initprovider.go b/cmd/initprovider.go
@@ -10,6 +10,7 @@ import (
 	"github.com/drakkan/sftpgo/v2/config"
 	"github.com/drakkan/sftpgo/v2/dataprovider"
 	"github.com/drakkan/sftpgo/v2/logger"
+	"github.com/drakkan/sftpgo/v2/service"
 	"github.com/drakkan/sftpgo/v2/util"
 )
 
@@ -40,13 +41,13 @@ Please take a look at the usage below to customize the options.`,
 			configDir = util.CleanDirInput(configDir)
 			err := config.LoadConfig(configDir, configFile)
 			if err != nil {
-				logger.WarnToConsole("Unable to initialize data provider, config load error: %v", err)
+				logger.ErrorToConsole("Unable to initialize data provider, config load error: %v", err)
 				return
 			}
 			kmsConfig := config.GetKMSConfig()
 			err = kmsConfig.Initialize()
 			if err != nil {
-				logger.ErrorToConsole("unable to initialize KMS: %v", err)
+				logger.ErrorToConsole("Unable to initialize KMS: %v", err)
 				os.Exit(1)
 			}
 			providerConf := config.GetProviderConf()
@@ -57,14 +58,27 @@ Please take a look at the usage below to customize the options.`,
 			} else if err == dataprovider.ErrNoInitRequired {
 				logger.InfoToConsole("%v", err.Error())
 			} else {
-				logger.WarnToConsole("Unable to initialize/update the data provider: %v", err)
+				logger.ErrorToConsole("Unable to initialize/update the data provider: %v", err)
 				os.Exit(1)
 			}
+			if providerConf.Driver != dataprovider.MemoryDataProviderName && loadDataFrom != "" {
+				service := service.Service{
+					LoadDataFrom:      loadDataFrom,
+					LoadDataMode:      loadDataMode,
+					LoadDataQuotaScan: loadDataQuotaScan,
+					LoadDataClean:     loadDataClean,
+				}
+				if err = service.LoadInitialData(); err != nil {
+					logger.ErrorToConsole("Cannot load initial data: %v", err)
+					os.Exit(1)
+				}
+			}
 		},
 	}
 )
 
 func init() {
 	rootCmd.AddCommand(initProviderCmd)
 	addConfigFlags(initProviderCmd)
+	addBaseLoadDataFlags(initProviderCmd)
 }
diff --git a/cmd/root.go b/cmd/root.go
@@ -126,6 +126,43 @@ env var too.`)
 	viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
 }
 
+func addBaseLoadDataFlags(cmd *cobra.Command) {
+	viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
+	viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
+	cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
+		`Load users and folders from this file.
+The file must be specified as absolute path
+and it must contain a backup obtained using
+the "dumpdata" REST API or compatible content.
+This flag can be set using SFTPGO_LOADDATA_FROM
+env var too.
+`)
+	viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
+	viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
+	cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
+		`Restore mode for data to load:
+  0 - new users are added, existing users are
+      updated
+  1 - New users are added, existing users are
+	  not modified
+This flag can be set using SFTPGO_LOADDATA_MODE
+env var too.
+`)
+	viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
+	viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
+	cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
+		`Determine if the loaddata-from file should
+be removed after a successful load. This flag
+can be set using SFTPGO_LOADDATA_CLEAN env var
+too. (default "false")
+`)
+	viper.BindPFlag(loadDataCleanKey, cmd.Flags().Lookup(loadDataCleanFlag)) //nolint:errcheck
+}
+
 func addServeFlags(cmd *cobra.Command) {
 	addConfigFlags(cmd)
 
@@ -192,30 +229,7 @@ using SFTPGO_LOG_UTC_TIME env var too.
 `)
 	viper.BindPFlag(logUTCTimeKey, cmd.Flags().Lookup(logUTCTimeFlag)) //nolint:errcheck
 
-	viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
-	viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
-	cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
-		`Load users and folders from this file.
-The file must be specified as absolute path
-and it must contain a backup obtained using
-the "dumpdata" REST API or compatible content.
-This flag can be set using SFTPGO_LOADDATA_FROM
-env var too.
-`)
-	viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
-
-	viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
-	viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
-	cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
-		`Restore mode for data to load:
-  0 - new users are added, existing users are
-      updated
-  1 - New users are added, existing users are
-	  not modified
-This flag can be set using SFTPGO_LOADDATA_MODE
-env var too.
-`)
-	viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
+	addBaseLoadDataFlags(cmd)
 
 	viper.SetDefault(loadDataQuotaScanKey, defaultLoadDataQuotaScan)
 	viper.BindEnv(loadDataQuotaScanKey, "SFTPGO_LOADDATA_QUOTA_SCAN") //nolint:errcheck
@@ -228,14 +242,4 @@ This flag can be set using SFTPGO_LOADDATA_QUOTA_SCAN
 env var too.
 (default 0)`)
 	viper.BindPFlag(loadDataQuotaScanKey, cmd.Flags().Lookup(loadDataQuotaScanFlag)) //nolint:errcheck
-
-	viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
-	viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
-	cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
-		`Determine if the loaddata-from file should
-be removed after a successful load. This flag
-can be set using SFTPGO_LOADDATA_CLEAN env var
-too. (default "false")
-`)
-	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
 }
diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go
@@ -804,6 +804,11 @@ func InitializeDatabase(cnf Config, basePath string) error {
 	} else {
 		credentialsDirPath = filepath.Join(basePath, config.CredentialsPath)
 	}
+	vfs.SetCredentialsDirPath(credentialsDirPath)
+
+	if err := initializeHashingAlgo(&cnf); err != nil {
+		return err
+	}
 
 	err := createProvider(basePath)
 	if err != nil {

diff --git a/docker/README.md b/docker/README.md
@@ -133,7 +133,7 @@ Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or
 
 Initial data can be loaded in the following ways:
 
-- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable
+- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable. This flag is supported for both the `serve` command (load initial data and start the service) and the `initprovider` command (initialize the provider, load initial data and exit)
 - by providing a dump file to the memory provider
 
 Please take a look [here](../docs/full-configuration.md) for more details.

diff --git a/go.mod b/go.mod
@@ -40,7 +40,7 @@ require (
 	github.com/prometheus/client_golang v1.12.1
 	github.com/rs/cors v1.8.2
 	github.com/rs/xid v1.3.0
-	github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1
+	github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672
 	github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c
 	github.com/shirou/gopsutil/v3 v3.22.1
 	github.com/spf13/afero v1.8.1
@@ -58,7 +58,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
-	golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
 	google.golang.org/api v0.70.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -120,7 +120,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect

diff --git a/go.sum b/go.sum
@@ -689,8 +689,8 @@ github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1 h1:n1Q4XjP7MrFJU2fC5CJqvtWU1HfNNkMiODLS5Of8wEA=
-github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
+github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672 h1:8tqGbO3HWm9kqGZxc8YLAND7QGJNppiwq+kMTpn8oOk=
+github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -741,8 +741,9 @@ github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9H
 github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
+github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
+github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
 github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
@@ -961,8 +962,8 @@ golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00=
-golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go
@@ -360,7 +360,7 @@ func RestoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota i
 			logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err)
 		}
 		if err != nil {
-			return fmt.Errorf("unable to restoreuser %#v: %w", user.Username, err)
+			return fmt.Errorf("unable to restore user %#v: %w", user.Username, err)
 		}
 		if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) {
 			if common.QuotaScans.AddUserQuotaScan(user.Username) {

diff --git a/logger/logger.go b/logger/logger.go
@@ -46,6 +46,10 @@ type StdLoggerWrapper struct {
 	Sender string
 }
 
+func init() {
+	zerolog.TimeFieldFormat = dateFormat
+}
+
 // Write implements the io.Writer interface. This is useful to set as a writer
 // for the standard library log.
 func (l *StdLoggerWrapper) Write(p []byte) (n int, err error) {
@@ -138,7 +142,6 @@ func GetLogger() *zerolog.Logger {
 
 // SetLogTime sets logging time related setting
 func SetLogTime(utc bool) {
-	zerolog.TimeFieldFormat = dateFormat
 	if utc {
 		zerolog.TimestampFunc = func() time.Time {
 			return time.Now().UTC()

diff --git a/service/service.go b/service/service.go
@@ -135,7 +135,7 @@ func (s *Service) Start() error {
 		}
 	}
 
-	err = s.loadInitialData()
+	err = s.LoadInitialData()
 	if err != nil {
 		logger.Error(logSender, "", "unable to load initial data: %v", err)
 		logger.ErrorToConsole("unable to load initial data: %v", err)
@@ -248,7 +248,8 @@ func (s *Service) Stop() {
 	logger.Debug(logSender, "", "Service stopped")
 }
 
-func (s *Service) loadInitialData() error {
+// LoadInitialData if a data file is set
+func (s *Service) LoadInitialData() error {
 	if s.LoadDataFrom == "" {
 		return nil
 	}
@@ -263,33 +264,26 @@ func (s *Service) loadInitialData() error {
 	}
 	info, err := os.Stat(s.LoadDataFrom)
 	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
-			logger.Warn(logSender, "", "unable to load initial data, the file %#v does not exist", s.LoadDataFrom)
-			logger.WarnToConsole("unable to load initial data, the file %#v does not exist", s.LoadDataFrom)
-			return nil
-		}
-		return err
+		return fmt.Errorf("unable to stat file %#v: %w", s.LoadDataFrom, err)
 	}
 	if info.Size() > httpd.MaxRestoreSize {
 		return fmt.Errorf("unable to restore input file %#v size too big: %v/%v bytes",
 			s.LoadDataFrom, info.Size(), httpd.MaxRestoreSize)
 	}
 	content, err := os.ReadFile(s.LoadDataFrom)
 	if err != nil {
-		return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err)
+		return fmt.Errorf("unable to read input file %#v: %w", s.LoadDataFrom, err)
 	}
 	dump, err := dataprovider.ParseDumpData(content)
 	if err != nil {
-		return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err)
+		return fmt.Errorf("unable to parse file to restore %#v: %w", s.LoadDataFrom, err)
 	}
 	err = s.restoreDump(&dump)
 	if err != nil {
 		return err
 	}
-	logger.Info(logSender, "", "data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom,
-		s.LoadDataMode, s.LoadDataQuotaScan)
-	logger.InfoToConsole("data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom,
-		s.LoadDataMode, s.LoadDataQuotaScan)
+	logger.Info(logSender, "", "data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode)
+	logger.InfoToConsole("data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode)
 	if s.LoadDataClean {
 		err = os.Remove(s.LoadDataFrom)
 		if err == nil {

diff --git a/util/util.go b/util/util.go
@@ -84,7 +84,7 @@ func RemoveDuplicates(obj []string) []string {
 
 // GetTimeAsMsSinceEpoch returns unix timestamp as milliseconds from a time struct
 func GetTimeAsMsSinceEpoch(t time.Time) int64 {
-	return t.UnixNano() / 1000000
+	return t.UnixMilli()
 }
 
 // GetTimeFromMsecSinceEpoch return a time struct from a unix timestamp with millisecond precision