diff --git a/.golangci.yml b/.golangci.yml index feb7e5a04fe49..3fd4e9d35e2b5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,8 @@ issues: exclude: - "Error return value of `io.WriteString` is not checked" # 'errcheck' errors in tools/dep_tree_resolver/go_deps.go - "Error return value of `pem.Encode` is not checked" # 'errcheck' errors in test/integration/utils/certificates.go + - "Error return value of `c.logErrorNotImplemented` is not checked" # 'errcheck' errors in pkg/config/nodetreemodel/config.go + - "Error return value of `n.logErrorNotImplemented` is not checked" # 'errcheck' errors in pkg/config/nodetreemodel/config.go - "exported: exported const Exec should have comment \\(or a comment on this block\\) or be unexported" # 'revive' errors in pkg/process/events/model/model_common.go - "exported: exported const APIName should have comment \\(or a comment on this block\\) or be unexported" # 'revive' errors in pkg/serverless/trace/inferredspan/constants.go - "unnecessary conversion" # 'unconvert' errors in test/integration/utils/certificates_test.go diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 2a509a0359fa5..7aa9b2720c67a 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -572,6 +572,7 @@ func (c *safeConfig) SetEnvKeyReplacer(r *strings.Replacer) { } // UnmarshalKey wraps Viper for concurrent access +// DEPRECATED: use pkg/config/structure.UnmarshalKey instead func (c *safeConfig) UnmarshalKey(key string, rawVal interface{}, opts ...viper.DecoderConfigOption) error { c.RLock() defer c.RUnlock() diff --git a/pkg/config/nodetreemodel/config.go b/pkg/config/nodetreemodel/config.go index 78f4a208680ff..4194c4f115006 100644 --- a/pkg/config/nodetreemodel/config.go +++ b/pkg/config/nodetreemodel/config.go @@ -7,12 +7,10 @@ package nodetreemodel import ( - "bytes" "errors" "fmt" "io" "os" - "path" "reflect" "strconv" "strings" @@ -30,7 +28,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -// sources list the known sources, following the order of hierarchy between them +// sources lists the known sources, following the order of hierarchy between them var sources = []model.Source{ model.SourceDefault, model.SourceUnknown, @@ -43,13 +41,15 @@ var sources = []model.Source{ model.SourceCLI, } -// safeConfig implements Config: -// - wraps viper with a safety lock -// - implements the additional DDHelpers -type safeConfig struct { - *viper.Viper - configSources map[model.Source]*viper.Viper +// ntmConfig implements Config +// - wraps a tree of node that represent config data +// - uses a lock to synchronize all methods +// - contains metadata about known keys, env var support +type ntmConfig struct { sync.RWMutex + root Node + noimpl notImplementedMethods + envPrefix string envKeyReplacer *strings.Replacer @@ -58,10 +58,17 @@ type safeConfig struct { // Proxy settings proxies *model.Proxy + configName string + configFile string + configType string + // configEnvVars is the set of env vars that are consulted for // configuration values. configEnvVars map[string]struct{} + // known keys are all the keys that meet at least one of these criteria: + // 1) have a default, 2) have an environment variable binded, 3) are an alias or 4) have been SetKnown() + knownKeys map[string]struct{} // keys that have been used but are unknown // used to warn (a single time) on use unknownKeys map[string]struct{} @@ -70,17 +77,29 @@ type safeConfig struct { extraConfigFilePaths []string } -// OnUpdate adds a callback to the list receivers to be called each time a value is changed in the configuration +// OnUpdate adds a callback to the list of receivers to be called each time a value is changed in the configuration // by a call to the 'Set' method. // Callbacks are only called if the value is effectively changed. -func (c *safeConfig) OnUpdate(callback model.NotificationReceiver) { +func (c *ntmConfig) OnUpdate(callback model.NotificationReceiver) { c.Lock() defer c.Unlock() c.notificationReceivers = append(c.notificationReceivers, callback) } -// Set wraps Viper for concurrent access -func (c *safeConfig) Set(key string, newValue interface{}, source model.Source) { +// getValue gets a value, should only be called within a locked mutex +func (c *ntmConfig) getValue(key string) (interface{}, error) { + return c.leafAtPath(key).GetAny() +} + +func (c *ntmConfig) setValueSource(key string, newValue interface{}, source model.Source) { + err := c.leafAtPath(key).SetWithSource(newValue, source) + if err != nil { + log.Errorf("%s", err) + } +} + +// Set assigns the newValue to the given key and marks it as originating from the given source +func (c *ntmConfig) Set(key string, newValue interface{}, source model.Source) { if source == model.SourceDefault { c.SetDefault(key, newValue) return @@ -89,9 +108,8 @@ func (c *safeConfig) Set(key string, newValue interface{}, source model.Source) // modify the config then release the lock to avoid deadlocks while notifying var receivers []model.NotificationReceiver c.Lock() - previousValue := c.Viper.Get(key) - c.configSources[source].Set(key, newValue) - c.mergeViperInstances(key) + previousValue, _ := c.getValue(key) + c.setValueSource(key, newValue, source) if !reflect.DeepEqual(previousValue, newValue) { // if the value has not changed, do not duplicate the slice so that no callback is called receivers = slices.Clone(c.notificationReceivers) @@ -104,66 +122,38 @@ func (c *safeConfig) Set(key string, newValue interface{}, source model.Source) } } -// SetWithoutSource sets the given value using source Unknown -func (c *safeConfig) SetWithoutSource(key string, value interface{}) { +// SetWithoutSource assigns the value to the given key using source Unknown +func (c *ntmConfig) SetWithoutSource(key string, value interface{}) { c.Set(key, value, model.SourceUnknown) } -// SetDefault wraps Viper for concurrent access -func (c *safeConfig) SetDefault(key string, value interface{}) { - c.Lock() - defer c.Unlock() - c.configSources[model.SourceDefault].Set(key, value) - c.Viper.SetDefault(key, value) +// SetDefault assigns the value to the given key using source Default +func (c *ntmConfig) SetDefault(key string, value interface{}) { + c.Set(key, value, model.SourceDefault) } // UnsetForSource unsets a config entry for a given source -func (c *safeConfig) UnsetForSource(key string, source model.Source) { - // modify the config then release the lock to avoid deadlocks while notifying - var receivers []model.NotificationReceiver +func (c *ntmConfig) UnsetForSource(_key string, _source model.Source) { c.Lock() - previousValue := c.Viper.Get(key) - c.configSources[source].Set(key, nil) - c.mergeViperInstances(key) - newValue := c.Viper.Get(key) // Can't use nil, so we get the newly computed value - if previousValue != nil { - // if the value has not changed, do not duplicate the slice so that no callback is called - receivers = slices.Clone(c.notificationReceivers) - } + c.logErrorNotImplemented("UnsetForSource") c.Unlock() - - // notifying all receiver about the updated setting - for _, receiver := range receivers { - receiver(key, previousValue, newValue) - } -} - -// mergeViperInstances is called after a change in an instance of Viper -// to recompute the state of the main Viper -// (it must be used with a lock to prevent concurrent access to Viper) -func (c *safeConfig) mergeViperInstances(key string) { - var val interface{} - for _, source := range sources { - if currVal := c.configSources[source].Get(key); currVal != nil { - val = currVal - } - } - c.Viper.Set(key, val) } // SetKnown adds a key to the set of known valid config keys -func (c *safeConfig) SetKnown(key string) { +func (c *ntmConfig) SetKnown(key string) { c.Lock() defer c.Unlock() - c.Viper.SetKnown(key) + key = strings.ToLower(key) + c.knownKeys[key] = struct{}{} } // IsKnown returns whether a key is known -func (c *safeConfig) IsKnown(key string) bool { +func (c *ntmConfig) IsKnown(key string) bool { c.RLock() defer c.RUnlock() - - return c.Viper.IsKnown(key) + key = strings.ToLower(key) + _, found := c.knownKeys[key] + return found } // checkKnownKey checks if a key is known, and if not logs a warning @@ -171,8 +161,8 @@ func (c *safeConfig) IsKnown(key string) bool { // // Must be called with the lock read-locked. // The lock can be released and re-locked. -func (c *safeConfig) checkKnownKey(key string) { - if c.Viper.IsKnown(key) { +func (c *ntmConfig) checkKnownKey(key string) { + if c.IsKnown(key) { return } @@ -180,224 +170,232 @@ func (c *safeConfig) checkKnownKey(key string) { return } - // need to write-lock to add the key to the unknownKeys map - c.RUnlock() - // but we need to have the lock in the same state (RLocked) at the end of the function - defer c.RLock() - - c.Lock() c.unknownKeys[key] = struct{}{} - c.Unlock() - - // log without holding the lock log.Warnf("config key %v is unknown", key) } // GetKnownKeysLowercased returns all the keys that meet at least one of these criteria: // 1) have a default, 2) have an environment variable binded or 3) have been SetKnown() // Note that it returns the keys lowercased. -func (c *safeConfig) GetKnownKeysLowercased() map[string]interface{} { +func (c *ntmConfig) GetKnownKeysLowercased() map[string]interface{} { c.RLock() defer c.RUnlock() // GetKnownKeysLowercased returns a fresh map, so the caller may do with it // as they please without holding the lock. - return c.Viper.GetKnownKeys() + ret := make(map[string]interface{}) + for key, value := range c.knownKeys { + ret[key] = value + } + return ret } -// ParseEnvAsStringSlice registers a transformer function to parse an an environment variables as a []string. -func (c *safeConfig) ParseEnvAsStringSlice(key string, fn func(string) []string) { +// ParseEnvAsStringSlice registers a transform function to parse an environment variable as a []string. +func (c *ntmConfig) ParseEnvAsStringSlice(key string, fn func(string) []string) { c.Lock() defer c.Unlock() - c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) + c.noimpl.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) } -// ParseEnvAsMapStringInterface registers a transformer function to parse an an environment variables as a -// map[string]interface{}. -func (c *safeConfig) ParseEnvAsMapStringInterface(key string, fn func(string) map[string]interface{}) { +// ParseEnvAsMapStringInterface registers a transform function to parse an environment variable as a map[string]interface{} +func (c *ntmConfig) ParseEnvAsMapStringInterface(key string, fn func(string) map[string]interface{}) { c.Lock() defer c.Unlock() - c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) + c.noimpl.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) } -// ParseEnvAsSliceMapString registers a transformer function to parse an an environment variables as a []map[string]string. -func (c *safeConfig) ParseEnvAsSliceMapString(key string, fn func(string) []map[string]string) { +// ParseEnvAsSliceMapString registers a transform function to parse an environment variable as a []map[string]string +func (c *ntmConfig) ParseEnvAsSliceMapString(key string, fn func(string) []map[string]string) { c.Lock() defer c.Unlock() - c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) + c.noimpl.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) } -// ParseEnvAsSlice registers a transformer function to parse an an environment variables as a -// []interface{}. -func (c *safeConfig) ParseEnvAsSlice(key string, fn func(string) []interface{}) { +// ParseEnvAsSlice registers a transform function to parse an environment variable as a []interface +func (c *ntmConfig) ParseEnvAsSlice(key string, fn func(string) []interface{}) { c.Lock() defer c.Unlock() - c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) + c.noimpl.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) } -// SetFs wraps Viper for concurrent access -func (c *safeConfig) SetFs(fs afero.Fs) { +// SetFs assigns a filesystem to the config +func (c *ntmConfig) SetFs(fs afero.Fs) { c.Lock() defer c.Unlock() - c.Viper.SetFs(fs) + c.noimpl.SetFs(fs) } -// IsSet wraps Viper for concurrent access -func (c *safeConfig) IsSet(key string) bool { +// IsSet checks if a key is set in the config +func (c *ntmConfig) IsSet(key string) bool { c.RLock() defer c.RUnlock() - return c.Viper.IsSet(key) + return c.noimpl.IsSet(key) } -func (c *safeConfig) AllKeysLowercased() []string { +// AllKeysLowercased returns all keys lower-cased +func (c *ntmConfig) AllKeysLowercased() []string { c.RLock() defer c.RUnlock() - return c.Viper.AllKeys() + return c.noimpl.AllKeys() +} + +func (c *ntmConfig) leafAtPath(key string) LeafNode { + pathParts := strings.Split(key, ".") + curr := c.root + for _, part := range pathParts { + next, err := curr.GetChild(part) + if err != nil { + return &missingLeaf + } + curr = next + } + if leaf, ok := curr.(LeafNode); ok { + return leaf + } + return &missingLeaf } -// Get wraps Viper for concurrent access -func (c *safeConfig) Get(key string) interface{} { +// Get returns a copy of the value for the given key +func (c *ntmConfig) Get(key string) interface{} { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetE(key) + val, err := c.getValue(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } + // NOTE: should only need to deepcopy for `Get`, because it can be an arbitrary value, + // and we shouldn't ever return complex types like maps and slices that could be modified + // by callers accidentally or on purpose. By copying, the caller may modify the result safetly return deepcopy.Copy(val) } // GetAllSources returns the value of a key for each source -func (c *safeConfig) GetAllSources(key string) []model.ValueWithSource { +func (c *ntmConfig) GetAllSources(key string) []model.ValueWithSource { c.RLock() defer c.RUnlock() c.checkKnownKey(key) vals := make([]model.ValueWithSource, len(sources)) - for i, source := range sources { - vals[i] = model.ValueWithSource{ - Source: source, - Value: deepcopy.Copy(c.configSources[source].Get(key)), - } - } + c.logErrorNotImplemented("GetAllSources") return vals } -// GetString wraps Viper for concurrent access -func (c *safeConfig) GetString(key string) string { +// GetString returns a string-typed value for the given key +func (c *ntmConfig) GetString(key string) string { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetStringE(key) + str, err := c.leafAtPath(key).GetString() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } - return val + return str } -// GetBool wraps Viper for concurrent access -func (c *safeConfig) GetBool(key string) bool { +// GetBool returns a bool-typed value for the given key +func (c *ntmConfig) GetBool(key string) bool { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetBoolE(key) + b, err := c.leafAtPath(key).GetBool() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } - return val + return b } -// GetInt wraps Viper for concurrent access -func (c *safeConfig) GetInt(key string) int { +// GetInt returns an int-typed value for the given key +func (c *ntmConfig) GetInt(key string) int { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetIntE(key) + val, err := c.leafAtPath(key).GetInt() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return val } -// GetInt32 wraps Viper for concurrent access -func (c *safeConfig) GetInt32(key string) int32 { +// GetInt32 returns an int32-typed value for the given key +func (c *ntmConfig) GetInt32(key string) int32 { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetInt32E(key) + val, err := c.leafAtPath(key).GetInt() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } - return val + return int32(val) } -// GetInt64 wraps Viper for concurrent access -func (c *safeConfig) GetInt64(key string) int64 { +// GetInt64 returns an int64-typed value for the given key +func (c *ntmConfig) GetInt64(key string) int64 { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetInt64E(key) + val, err := c.leafAtPath(key).GetInt() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } - return val + return int64(val) } -// GetFloat64 wraps Viper for concurrent access -func (c *safeConfig) GetFloat64(key string) float64 { +// GetFloat64 returns a float64-typed value for the given key +func (c *ntmConfig) GetFloat64(key string) float64 { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetFloat64E(key) + val, err := c.leafAtPath(key).GetFloat() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return val } -// GetTime wraps Viper for concurrent access -func (c *safeConfig) GetTime(key string) time.Time { +// GetTime returns a time-typed value for the given key +func (c *ntmConfig) GetTime(key string) time.Time { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetTimeE(key) + val, err := c.leafAtPath(key).GetTime() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return val } -// GetDuration wraps Viper for concurrent access -func (c *safeConfig) GetDuration(key string) time.Duration { +// GetDuration returns a duration-typed value for the given key +func (c *ntmConfig) GetDuration(key string) time.Duration { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetDurationE(key) + val, err := c.leafAtPath(key).GetDuration() if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return val } -// GetStringSlice wraps Viper for concurrent access -func (c *safeConfig) GetStringSlice(key string) []string { +// GetStringSlice returns a string slice value for the given key +func (c *ntmConfig) GetStringSlice(key string) []string { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetStringSliceE(key) + val, err := c.noimpl.GetStringSliceE(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return slices.Clone(val) } -// GetFloat64SliceE loads a key as a []float64 -func (c *safeConfig) GetFloat64SliceE(key string) ([]float64, error) { +// GetFloat64SliceE returns a float slice value for the given key, or an error +func (c *ntmConfig) GetFloat64SliceE(key string) ([]float64, error) { c.RLock() defer c.RUnlock() c.checkKnownKey(key) // We're using GetStringSlice because viper can only parse list of string from env variables - list, err := c.Viper.GetStringSliceE(key) + list, err := c.noimpl.GetStringSliceE(key) if err != nil { return nil, fmt.Errorf("'%v' is not a list", key) } @@ -413,86 +411,83 @@ func (c *safeConfig) GetFloat64SliceE(key string) ([]float64, error) { return res, nil } -// GetStringMap wraps Viper for concurrent access -func (c *safeConfig) GetStringMap(key string) map[string]interface{} { +// GetStringMap returns a map[string]interface value for the given key +func (c *ntmConfig) GetStringMap(key string) map[string]interface{} { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetStringMapE(key) + val, err := c.noimpl.GetStringMapE(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return deepcopy.Copy(val).(map[string]interface{}) } -// GetStringMapString wraps Viper for concurrent access -func (c *safeConfig) GetStringMapString(key string) map[string]string { +// GetStringMapString returns a map[string]string value for the given key +func (c *ntmConfig) GetStringMapString(key string) map[string]string { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetStringMapStringE(key) + val, err := c.noimpl.GetStringMapStringE(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return deepcopy.Copy(val).(map[string]string) } -// GetStringMapStringSlice wraps Viper for concurrent access -func (c *safeConfig) GetStringMapStringSlice(key string) map[string][]string { +// GetStringMapStringSlice returns a map[string][]string value for the given key +func (c *ntmConfig) GetStringMapStringSlice(key string) map[string][]string { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetStringMapStringSliceE(key) + val, err := c.noimpl.GetStringMapStringSliceE(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return deepcopy.Copy(val).(map[string][]string) } -// GetSizeInBytes wraps Viper for concurrent access -func (c *safeConfig) GetSizeInBytes(key string) uint { +// GetSizeInBytes returns the size in bytes of the filename for the given key +func (c *ntmConfig) GetSizeInBytes(key string) uint { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - val, err := c.Viper.GetSizeInBytesE(key) + val, err := c.noimpl.GetSizeInBytesE(key) if err != nil { log.Warnf("failed to get configuration value for key %q: %s", key, err) } return val } -// GetSource wraps Viper for concurrent access -func (c *safeConfig) GetSource(key string) model.Source { +func (c *ntmConfig) logErrorNotImplemented(method string) error { + err := fmt.Errorf("not implemented: %s", method) + log.Error(err) + return err +} + +// GetSource returns the source of the given key +func (c *ntmConfig) GetSource(key string) model.Source { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - var source model.Source - for _, s := range sources { - if c.configSources[s].Get(key) != nil { - source = s - } - } - return source + c.logErrorNotImplemented("GetSource") + return model.SourceUnknown } -// SetEnvPrefix wraps Viper for concurrent access, and keeps the envPrefix for -// future reference -func (c *safeConfig) SetEnvPrefix(in string) { +// SetEnvPrefix sets the environment variable prefix to use +func (c *ntmConfig) SetEnvPrefix(in string) { c.Lock() defer c.Unlock() - c.configSources[model.SourceEnvVar].SetEnvPrefix(in) - c.Viper.SetEnvPrefix(in) c.envPrefix = in } -// mergeWithEnvPrefix derives the environment variable that Viper will use for a given key. -// mergeWithEnvPrefix must be called while holding the config log (read or write). -func (c *safeConfig) mergeWithEnvPrefix(key string) string { +// mergeWithEnvPrefix derives the environment variable to use for a given key. +func (c *ntmConfig) mergeWithEnvPrefix(key string) string { return strings.Join([]string{c.envPrefix, strings.ToUpper(key)}, "_") } -// BindEnv wraps Viper for concurrent access, and adds tracking of the configurable env vars -func (c *safeConfig) BindEnv(key string, envvars ...string) { +// BindEnv binds one or more environment variables to the given key +func (c *ntmConfig) BindEnv(key string, envvars ...string) { c.Lock() defer c.Unlock() var envKeys []string @@ -513,85 +508,30 @@ func (c *safeConfig) BindEnv(key string, envvars ...string) { c.configEnvVars[key] = struct{}{} } - newKeys := append([]string{key}, envvars...) - _ = c.configSources[model.SourceEnvVar].BindEnv(newKeys...) - _ = c.Viper.BindEnv(newKeys...) + c.logErrorNotImplemented("BindEnv") } -// SetEnvKeyReplacer wraps Viper for concurrent access -func (c *safeConfig) SetEnvKeyReplacer(r *strings.Replacer) { +// SetEnvKeyReplacer binds a replacer function for keys +func (c *ntmConfig) SetEnvKeyReplacer(_r *strings.Replacer) { c.Lock() defer c.Unlock() - c.configSources[model.SourceEnvVar].SetEnvKeyReplacer(r) - c.Viper.SetEnvKeyReplacer(r) - c.envKeyReplacer = r + c.logErrorNotImplemented("SetEnvKeyReplacer") } -// UnmarshalKey wraps Viper for concurrent access -func (c *safeConfig) UnmarshalKey(key string, rawVal interface{}, opts ...viper.DecoderConfigOption) error { +// UnmarshalKey unmarshals the data for the given key +// DEPRECATED: use pkg/config/structure.UnmarshalKey instead +func (c *ntmConfig) UnmarshalKey(key string, _rawVal interface{}, _opts ...viper.DecoderConfigOption) error { c.RLock() defer c.RUnlock() c.checkKnownKey(key) - return c.Viper.UnmarshalKey(key, rawVal, opts...) + return c.logErrorNotImplemented("UnmarshalKey") } -// ReadInConfig wraps Viper for concurrent access -func (c *safeConfig) ReadInConfig() error { +// MergeConfig merges in another config +func (c *ntmConfig) MergeConfig(_in io.Reader) error { c.Lock() defer c.Unlock() - - // ReadInConfig reset configuration with the main config file - err := errors.Join(c.Viper.ReadInConfig(), c.configSources[model.SourceFile].ReadInConfig()) - if err != nil { - return err - } - - type extraConf struct { - path string - content []byte - } - - // Read extra config files - extraConfContents := []extraConf{} - for _, path := range c.extraConfigFilePaths { - b, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("could not read extra config file '%s': %w", path, err) - } - extraConfContents = append(extraConfContents, extraConf{path: path, content: b}) - } - - // Merge with base config and 'file' config - for _, confFile := range extraConfContents { - err = errors.Join(c.Viper.MergeConfig(bytes.NewReader(confFile.content)), c.configSources[model.SourceFile].MergeConfig(bytes.NewReader(confFile.content))) - if err != nil { - return fmt.Errorf("error merging %s config file: %w", confFile.path, err) - } - log.Infof("extra configuration file %s was loaded successfully", confFile.path) - } - return nil -} - -// ReadConfig wraps Viper for concurrent access -func (c *safeConfig) ReadConfig(in io.Reader) error { - c.Lock() - defer c.Unlock() - b, err := io.ReadAll(in) - if err != nil { - return err - } - err = c.Viper.ReadConfig(bytes.NewReader(b)) - if err != nil { - return err - } - return c.configSources[model.SourceFile].ReadConfig(bytes.NewReader(b)) -} - -// MergeConfig wraps Viper for concurrent access -func (c *safeConfig) MergeConfig(in io.Reader) error { - c.Lock() - defer c.Unlock() - return c.Viper.MergeConfig(in) + return c.logErrorNotImplemented("MergeConfig") } // MergeFleetPolicy merges the configuration from the reader given with an existing config @@ -599,7 +539,7 @@ func (c *safeConfig) MergeConfig(in io.Reader) error { // according to sources priority order. // // Note: this should only be called at startup, as notifiers won't receive a notification when this loads -func (c *safeConfig) MergeFleetPolicy(configPath string) error { +func (c *ntmConfig) MergeFleetPolicy(configPath string) error { c.Lock() defer c.Unlock() @@ -616,83 +556,57 @@ func (c *safeConfig) MergeFleetPolicy(configPath string) error { } defer in.Close() - c.configSources[model.SourceFleetPolicies].SetConfigType("yaml") - err = c.configSources[model.SourceFleetPolicies].MergeConfigOverride(in) - if err != nil { - return err - } - for _, key := range c.configSources[model.SourceFleetPolicies].AllKeys() { - c.mergeViperInstances(key) - } - log.Infof("Fleet policies configuration %s successfully merged", path.Base(configPath)) - return nil + // TODO: Implement merging, merge in the policy that was read + return c.logErrorNotImplemented("MergeFleetPolicy") } // MergeConfigMap merges the configuration from the map given with an existing config. // Note that the map given may be modified. -func (c *safeConfig) MergeConfigMap(cfg map[string]any) error { +func (c *ntmConfig) MergeConfigMap(_cfg map[string]any) error { c.Lock() defer c.Unlock() - return c.Viper.MergeConfigMap(cfg) + c.logErrorNotImplemented("AllSettings") + return nil } -// AllSettings wraps Viper for concurrent access -func (c *safeConfig) AllSettings() map[string]interface{} { +// AllSettings returns all settings from the config +func (c *ntmConfig) AllSettings() map[string]interface{} { c.RLock() defer c.RUnlock() - - // AllSettings returns a fresh map, so the caller may do with it - // as they please without holding the lock. - return c.Viper.AllSettings() + c.logErrorNotImplemented("AllSettings") + return nil } // AllSettingsWithoutDefault returns a copy of the all the settings in the configuration without defaults -func (c *safeConfig) AllSettingsWithoutDefault() map[string]interface{} { +func (c *ntmConfig) AllSettingsWithoutDefault() map[string]interface{} { c.RLock() defer c.RUnlock() - - // AllSettingsWithoutDefault returns a fresh map, so the caller may do with it - // as they please without holding the lock. - return c.Viper.AllSettingsWithoutDefault() + c.logErrorNotImplemented("AllSettingsWithoutDefault") + return nil } // AllSettingsBySource returns the settings from each source (file, env vars, ...) -func (c *safeConfig) AllSettingsBySource() map[model.Source]interface{} { +func (c *ntmConfig) AllSettingsBySource() map[model.Source]interface{} { c.RLock() defer c.RUnlock() - sources := []model.Source{ - model.SourceDefault, - model.SourceUnknown, - model.SourceFile, - model.SourceEnvVar, - model.SourceFleetPolicies, - model.SourceAgentRuntime, - model.SourceRC, - model.SourceCLI, - model.SourceLocalConfigProcess, - } res := map[model.Source]interface{}{} - for _, source := range sources { - res[source] = c.configSources[source].AllSettingsWithoutDefault() - } - res[model.SourceProvided] = c.Viper.AllSettingsWithoutDefault() + c.logErrorNotImplemented("AllSettingsBySource") return res } -// AddConfigPath wraps Viper for concurrent access -func (c *safeConfig) AddConfigPath(in string) { +// AddConfigPath adds another config for the given path +func (c *ntmConfig) AddConfigPath(_in string) { c.Lock() defer c.Unlock() - c.configSources[model.SourceFile].AddConfigPath(in) - c.Viper.AddConfigPath(in) + c.logErrorNotImplemented("AddConfigPath") } // AddExtraConfigPaths allows adding additional configuration files // which will be merged into the main configuration during the ReadInConfig call. // Configuration files are merged sequentially. If a key already exists and the foreign value type matches the existing one, the foreign value overrides it. // If both the existing value and the new value are nested configurations, they are merged recursively following the same principles. -func (c *safeConfig) AddExtraConfigPaths(ins []string) error { +func (c *ntmConfig) AddExtraConfigPaths(ins []string) error { if len(ins) == 0 { return nil } @@ -717,48 +631,44 @@ func (c *safeConfig) AddExtraConfigPaths(ins []string) error { return err } -// SetConfigName wraps Viper for concurrent access -func (c *safeConfig) SetConfigName(in string) { +// SetConfigName sets the name of the config +func (c *ntmConfig) SetConfigName(in string) { c.Lock() defer c.Unlock() - c.configSources[model.SourceFile].SetConfigName(in) - c.Viper.SetConfigName(in) + c.configName = in + c.configFile = "" } -// SetConfigFile wraps Viper for concurrent access -func (c *safeConfig) SetConfigFile(in string) { +// SetConfigFile sets the config file +func (c *ntmConfig) SetConfigFile(in string) { c.Lock() defer c.Unlock() - c.configSources[model.SourceFile].SetConfigFile(in) - c.Viper.SetConfigFile(in) + c.configFile = in } -// SetConfigType wraps Viper for concurrent access -func (c *safeConfig) SetConfigType(in string) { +// SetConfigType sets the type of the config +func (c *ntmConfig) SetConfigType(in string) { c.Lock() defer c.Unlock() - c.configSources[model.SourceFile].SetConfigType(in) - c.Viper.SetConfigType(in) + c.configType = in } -// ConfigFileUsed wraps Viper for concurrent access -func (c *safeConfig) ConfigFileUsed() string { +// ConfigFileUsed returns the config file +func (c *ntmConfig) ConfigFileUsed() string { c.RLock() defer c.RUnlock() - return c.Viper.ConfigFileUsed() + return c.configFile } -func (c *safeConfig) SetTypeByDefaultValue(in bool) { +// SetTypeByDefaultValue enables typing using default values +func (c *ntmConfig) SetTypeByDefaultValue(_in bool) { c.Lock() defer c.Unlock() - for _, source := range sources { - c.configSources[source].SetTypeByDefaultValue(in) - } - c.Viper.SetTypeByDefaultValue(in) + c.logErrorNotImplemented("SetTypeByDefaultValue") } -// GetEnvVars implements the Config interface -func (c *safeConfig) GetEnvVars() []string { +// GetEnvVars gets all environment variables +func (c *ntmConfig) GetEnvVars() []string { c.RLock() defer c.RUnlock() vars := make([]string, 0, len(c.configEnvVars)) @@ -768,34 +678,31 @@ func (c *safeConfig) GetEnvVars() []string { return vars } -// BindEnvAndSetDefault implements the Config interface -func (c *safeConfig) BindEnvAndSetDefault(key string, val interface{}, envvars ...string) { +// BindEnvAndSetDefault binds an environment variable and sets a default for the given key +func (c *ntmConfig) BindEnvAndSetDefault(key string, val interface{}, envvars ...string) { c.SetDefault(key, val) c.BindEnv(key, envvars...) //nolint:errcheck } -func (c *safeConfig) Warnings() *model.Warnings { +// Warnings just returns nil +func (c *ntmConfig) Warnings() *model.Warnings { return nil } -func (c *safeConfig) Object() model.Reader { +// Object returns the config as a Reader interface +func (c *ntmConfig) Object() model.Reader { return c } // NewConfig returns a new Config object. func NewConfig(name string, envPrefix string, envKeyReplacer *strings.Replacer) model.Config { - config := safeConfig{ - Viper: viper.New(), - configSources: map[model.Source]*viper.Viper{}, + config := ntmConfig{ + noimpl: ¬ImplMethodsImpl{}, configEnvVars: map[string]struct{}{}, + knownKeys: map[string]struct{}{}, unknownKeys: map[string]struct{}{}, } - // load one Viper instance per source of setting change - for _, source := range sources { - config.configSources[source] = viper.New() - } - config.SetTypeByDefaultValue(true) config.SetConfigName(name) config.SetEnvPrefix(envPrefix) @@ -806,13 +713,13 @@ func NewConfig(name string, envPrefix string, envKeyReplacer *strings.Replacer) // CopyConfig copies the given config to the receiver config. This should only be used in tests as replacing // the global config reference is unsafe. -func (c *safeConfig) CopyConfig(cfg model.Config) { +func (c *ntmConfig) CopyConfig(cfg model.Config) { c.Lock() defer c.Unlock() - - if cfg, ok := cfg.(*safeConfig); ok { - c.Viper = cfg.Viper - c.configSources = cfg.configSources + c.logErrorNotImplemented("CopyConfig") + if cfg, ok := cfg.(*ntmConfig); ok { + // TODO: Probably a bug, should be a deep copy, add a test and verify + c.root = cfg.root c.envPrefix = cfg.envPrefix c.envKeyReplacer = cfg.envKeyReplacer c.proxies = cfg.proxies @@ -821,33 +728,34 @@ func (c *safeConfig) CopyConfig(cfg model.Config) { c.notificationReceivers = cfg.notificationReceivers return } - panic("Replacement config must be an instance of safeConfig") + panic("Replacement config must be an instance of ntmConfig") } // GetProxies returns the proxy settings from the configuration -func (c *safeConfig) GetProxies() *model.Proxy { +func (c *ntmConfig) GetProxies() *model.Proxy { c.Lock() - defer c.Unlock() - if c.proxies != nil { - return c.proxies + hasProxies := c.proxies + c.Unlock() + if hasProxies != nil { + return hasProxies } - if c.Viper.GetBool("fips.enabled") { + if c.GetBool("fips.enabled") { return nil } - if !c.Viper.IsSet("proxy.http") && !c.Viper.IsSet("proxy.https") && !c.Viper.IsSet("proxy.no_proxy") { + if !c.IsSet("proxy.http") && !c.IsSet("proxy.https") && !c.IsSet("proxy.no_proxy") { return nil } p := &model.Proxy{ - HTTP: c.Viper.GetString("proxy.http"), - HTTPS: c.Viper.GetString("proxy.https"), - NoProxy: c.Viper.GetStringSlice("proxy.no_proxy"), + HTTP: c.GetString("proxy.http"), + HTTPS: c.GetString("proxy.https"), + NoProxy: c.GetStringSlice("proxy.no_proxy"), } - c.proxies = p return c.proxies } -func (c *safeConfig) ExtraConfigFilesUsed() []string { +// ExtraConfigFilesUsed returns the additional config files used +func (c *ntmConfig) ExtraConfigFilesUsed() []string { c.Lock() defer c.Unlock() res := make([]string, len(c.extraConfigFilePaths)) diff --git a/pkg/config/nodetreemodel/go.mod b/pkg/config/nodetreemodel/go.mod index 1ae60bca98684..68a7ed4848cb1 100644 --- a/pkg/config/nodetreemodel/go.mod +++ b/pkg/config/nodetreemodel/go.mod @@ -14,22 +14,26 @@ require ( github.com/DataDog/viper v1.13.5 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/spf13/afero v1.11.0 + github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 + gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.4.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.3 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/config/nodetreemodel/missing.go b/pkg/config/nodetreemodel/missing.go new file mode 100644 index 0000000000000..79d470b702a3f --- /dev/null +++ b/pkg/config/nodetreemodel/missing.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "fmt" + "time" + + "github.com/DataDog/datadog-agent/pkg/config/model" +) + +// missingLeafImpl is a none-object representing when a child node is missing +type missingLeafImpl struct{} + +var missingLeaf missingLeafImpl + +func (m *missingLeafImpl) GetAny() (any, error) { + return nil, fmt.Errorf("GetAny(): missing") +} + +func (m *missingLeafImpl) GetBool() (bool, error) { + return false, fmt.Errorf("GetBool(): missing") +} + +func (m *missingLeafImpl) GetInt() (int, error) { + return 0, fmt.Errorf("GetInt(): missing") +} + +func (m *missingLeafImpl) GetFloat() (float64, error) { + return 0.0, fmt.Errorf("GetFloat(): missing") +} + +func (m *missingLeafImpl) GetString() (string, error) { + return "", fmt.Errorf("GetString(): missing") +} + +func (m *missingLeafImpl) GetTime() (time.Time, error) { + return time.Time{}, fmt.Errorf("GetTime(): missing") +} + +func (m *missingLeafImpl) GetDuration() (time.Duration, error) { + return time.Duration(0), fmt.Errorf("GetDuration(): missing") +} + +func (m *missingLeafImpl) SetWithSource(interface{}, model.Source) error { + return fmt.Errorf("SetWithSource(): missing") +} diff --git a/pkg/config/nodetreemodel/node.go b/pkg/config/nodetreemodel/node.go new file mode 100644 index 0000000000000..3687bf24bc1fc --- /dev/null +++ b/pkg/config/nodetreemodel/node.go @@ -0,0 +1,376 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "fmt" + "slices" + "strconv" + "strings" + "time" + + "github.com/DataDog/datadog-agent/pkg/config/model" + "golang.org/x/exp/maps" +) + +// ErrNotFound is an error for when a key is not found +var ErrNotFound = fmt.Errorf("not found") + +// NewNode constructs a Node from either a map, a slice, or a scalar value +func NewNode(v interface{}) (Node, error) { + switch it := v.(type) { + case map[interface{}]interface{}: + return newMapNodeImpl(mapInterfaceToMapString(it)) + case map[string]interface{}: + return newMapNodeImpl(it) + case []interface{}: + return newArrayNodeImpl(it) + } + if isScalar(v) { + return newLeafNodeImpl(v) + } + // Finally, try determining node type using reflection, should only be needed for unit tests that + // supply data that isn't one of the "plain" types produced by parsing json, yaml, etc + node, err := asReflectionNode(v) + if err == errUnknownConversion { + return nil, fmt.Errorf("could not create node from: %v of type %T", v, v) + } + return node, err +} + +// LeafNode represents a leaf node of the config +type LeafNode interface { + GetAny() (interface{}, error) + GetBool() (bool, error) + GetInt() (int, error) + GetFloat() (float64, error) + GetString() (string, error) + GetTime() (time.Time, error) + GetDuration() (time.Duration, error) + SetWithSource(interface{}, model.Source) error +} + +// ArrayNode represents a node with ordered, numerically indexed set of children +type ArrayNode interface { + Size() int + Index(int) (Node, error) +} + +// Node represents an arbitrary node +type Node interface { + GetChild(string) (Node, error) + ChildrenKeys() ([]string, error) +} + +// leafNode represents a leaf with a scalar value + +type leafNodeImpl struct { + // val must be a scalar kind + val interface{} + source model.Source +} + +func newLeafNodeImpl(v interface{}) (Node, error) { + if isScalar(v) { + return &leafNodeImpl{val: v}, nil + } + return nil, fmt.Errorf("cannot create leaf node from %v of type %T", v, v) +} + +var _ LeafNode = (*leafNodeImpl)(nil) +var _ Node = (*leafNodeImpl)(nil) + +// arrayNode represents a node with an ordered array of children + +type arrayNodeImpl struct { + nodes []Node +} + +func newArrayNodeImpl(v []interface{}) (Node, error) { + nodes := make([]Node, 0, len(v)) + for _, it := range v { + if n, ok := it.(Node); ok { + nodes = append(nodes, n) + continue + } + n, err := NewNode(it) + if err != nil { + return nil, err + } + nodes = append(nodes, n) + } + return &arrayNodeImpl{nodes: nodes}, nil +} + +var _ ArrayNode = (*arrayNodeImpl)(nil) +var _ Node = (*arrayNodeImpl)(nil) + +// node represents an arbitrary node of the tree + +type mapNodeImpl struct { + val map[string]interface{} + // remapCase maps each lower-case key to the original case. This + // enables GetChild to retrieve values using case-insensitive keys + remapCase map[string]string +} + +func newMapNodeImpl(v map[string]interface{}) (Node, error) { + return &mapNodeImpl{val: v, remapCase: makeRemapCase(v)}, nil +} + +var _ Node = (*mapNodeImpl)(nil) + +/////// + +func isScalar(v interface{}) bool { + switch v.(type) { + case int, int8, int16, int32, int64: + return true + case uint, uint8, uint16, uint32, uint64: + return true + case bool, string, float32, float64, time.Time, time.Duration: + return true + default: + return false + } +} + +// creates a map that converts keys from their lower-cased version to their original case +func makeRemapCase(m map[string]interface{}) map[string]string { + remap := make(map[string]string) + for k := range m { + remap[strings.ToLower(k)] = k + } + return remap +} + +func mapInterfaceToMapString(m map[interface{}]interface{}) map[string]interface{} { + res := make(map[string]interface{}, len(m)) + for k, v := range m { + mk := "" + if str, ok := k.(string); ok { + mk = str + } else { + mk = fmt.Sprintf("%s", k) + } + res[mk] = v + } + return res +} + +///// + +// GetChild returns the child node at the given case-insensitive key, or an error if not found +func (n *mapNodeImpl) GetChild(key string) (Node, error) { + mkey := n.remapCase[strings.ToLower(key)] + child, found := n.val[mkey] + if !found { + return nil, ErrNotFound + } + // If the map is already storing a Node, return it + if n, ok := child.(Node); ok { + return n, nil + } + // Otherwise construct a new node + return NewNode(child) +} + +// ChildrenKeys returns the list of keys of the children of the given node, if it is a map +func (n *mapNodeImpl) ChildrenKeys() ([]string, error) { + mapkeys := maps.Keys(n.val) + // map keys are iterated non-deterministically, sort them + slices.Sort(mapkeys) + return mapkeys, nil +} + +// GetChild returns an error because array node does not have children accessible by name +func (n *arrayNodeImpl) GetChild(string) (Node, error) { + return nil, fmt.Errorf("arrayNodeImpl.GetChild not implemented") +} + +// ChildrenKeys returns an error because array node does not have children accessible by name +func (n *arrayNodeImpl) ChildrenKeys() ([]string, error) { + return nil, fmt.Errorf("arrayNodeImpl.ChildrenKeys not implemented") +} + +// Size returns number of children in the list +func (n *arrayNodeImpl) Size() int { + return len(n.nodes) +} + +// Index returns the kth element of the list +func (n *arrayNodeImpl) Index(k int) (Node, error) { + if k < 0 || k >= len(n.nodes) { + return nil, ErrNotFound + } + return n.nodes[k], nil +} + +// GetChild returns an error because a leaf has no children +func (n *leafNodeImpl) GetChild(key string) (Node, error) { + return nil, fmt.Errorf("can't GetChild(%s) of a leaf node", key) +} + +// ChildrenKeys returns an error because a leaf has no children +func (n *leafNodeImpl) ChildrenKeys() ([]string, error) { + return nil, fmt.Errorf("can't get ChildrenKeys of a leaf node") +} + +// GetAny returns the scalar as an interface +func (n *leafNodeImpl) GetAny() (interface{}, error) { + return n, nil +} + +// GetBool returns the scalar as a bool, or an error otherwise +func (n *leafNodeImpl) GetBool() (bool, error) { + switch it := n.val.(type) { + case bool: + return it, nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + num, err := n.GetInt() + if err != nil { + return false, err + } + return num != 0, nil + case string: + return convertToBool(it) + default: + return false, newConversionError(n, "bool") + } +} + +// GetInt returns the scalar as a int, or an error otherwise +func (n *leafNodeImpl) GetInt() (int, error) { + switch it := n.val.(type) { + case int: + return int(it), nil + case int8: + return int(it), nil + case int16: + return int(it), nil + case int32: + return int(it), nil + case int64: + return int(it), nil + case uint: + return int(it), nil + case uint8: + return int(it), nil + case uint16: + return int(it), nil + case uint32: + return int(it), nil + case uint64: + return int(it), nil + case float32: + return int(it), nil + case float64: + return int(it), nil + } + return 0, newConversionError(n.val, "int") +} + +// GetFloat returns the scalar as a float64, or an error otherwise +func (n *leafNodeImpl) GetFloat() (float64, error) { + switch it := n.val.(type) { + case int: + return float64(it), nil + case int8: + return float64(it), nil + case int16: + return float64(it), nil + case int32: + return float64(it), nil + case int64: + return float64(it), nil + case uint: + return float64(it), nil + case uint8: + return float64(it), nil + case uint16: + return float64(it), nil + case uint32: + return float64(it), nil + case uint64: + return float64(it), nil + case float32: + return float64(it), nil + case float64: + return float64(it), nil + } + return 0, newConversionError(n.val, "float") +} + +// GetString returns the scalar as a string, or an error otherwise +func (n *leafNodeImpl) GetString() (string, error) { + switch it := n.val.(type) { + case int, int8, int16, int32, int64: + num, err := n.GetInt() + if err != nil { + return "", err + } + stringVal := strconv.FormatInt(int64(num), 10) + return stringVal, nil + case uint, uint8, uint16, uint32, uint64: + num, err := n.GetInt() + if err != nil { + return "", err + } + stringVal := strconv.FormatUint(uint64(num), 10) + return stringVal, nil + case float32: + f, err := n.GetFloat() + if err != nil { + return "", err + } + stringVal := strconv.FormatFloat(f, 'f', -1, 32) + return stringVal, nil + case float64: + f, err := n.GetFloat() + if err != nil { + return "", err + } + stringVal := strconv.FormatFloat(f, 'f', -1, 64) + return stringVal, nil + case string: + return it, nil + } + return "", newConversionError(n.val, "string") +} + +// GetTime returns the scalar as a time, or an error otherwise, not implemented +func (n *leafNodeImpl) GetTime() (time.Time, error) { + return time.Time{}, fmt.Errorf("not implemented") +} + +// GetDuration returns the scalar as a duration, or an error otherwise, not implemented +func (n *leafNodeImpl) GetDuration() (time.Duration, error) { + return time.Duration(0), fmt.Errorf("not implemented") +} + +// Set assigns a value in the config, for the given source +func (n *leafNodeImpl) SetWithSource(newValue interface{}, source model.Source) error { + // TODO: enforce type-checking, return an error if type changes + n.val = newValue + n.source = source + // TODO: Record previous value and source + return nil +} + +// convert a string to a bool using standard yaml constants +func convertToBool(text string) (bool, error) { + lower := strings.ToLower(text) + if lower == "y" || lower == "yes" || lower == "on" || lower == "true" || lower == "1" { + return true, nil + } else if lower == "n" || lower == "no" || lower == "off" || lower == "false" || lower == "0" { + return false, nil + } + return false, newConversionError(text, "bool") +} + +func newConversionError(v interface{}, expectType string) error { + return fmt.Errorf("could not convert to %s: %v of type %T", expectType, v, v) +} diff --git a/pkg/config/nodetreemodel/node_test.go b/pkg/config/nodetreemodel/node_test.go new file mode 100644 index 0000000000000..7cf34c9ceb1be --- /dev/null +++ b/pkg/config/nodetreemodel/node_test.go @@ -0,0 +1,66 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewNodeAndNodeMethods(t *testing.T) { + obj := map[string]interface{}{ + "a": "apple", + "b": 123, + "c": map[string]interface{}{ + "d": true, + "e": []string{"f", "g"}, + }, + } + + n, err := NewNode(obj) + assert.NoError(t, err) + + keys, err := n.ChildrenKeys() + assert.NoError(t, err) + assert.Equal(t, keys, []string{"a", "b", "c"}) + + first, err := n.GetChild("a") + assert.NoError(t, err) + + firstLeaf := first.(LeafNode) + str, err := firstLeaf.GetString() + assert.NoError(t, err) + assert.Equal(t, str, "apple") + + second, err := n.GetChild("b") + assert.NoError(t, err) + + secondLeaf := second.(LeafNode) + num, err := secondLeaf.GetInt() + assert.NoError(t, err) + assert.Equal(t, num, 123) + + third, err := n.GetChild("c") + assert.NoError(t, err) + _, ok := third.(LeafNode) + assert.Equal(t, ok, false) + + keys, err = third.ChildrenKeys() + assert.NoError(t, err) + assert.Equal(t, keys, []string{"d", "e"}) + + fourth, err := third.GetChild("d") + assert.NoError(t, err) + + fourthLeaf := fourth.(LeafNode) + b, err := fourthLeaf.GetBool() + assert.NoError(t, err) + assert.Equal(t, b, true) + + _, err = third.GetChild("e") + assert.NoError(t, err) +} diff --git a/pkg/config/nodetreemodel/noimpl_methods.go b/pkg/config/nodetreemodel/noimpl_methods.go new file mode 100644 index 0000000000000..2bfa2cfb98c9f --- /dev/null +++ b/pkg/config/nodetreemodel/noimpl_methods.go @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "fmt" + + "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/spf13/afero" +) + +type notImplementedMethods interface { + SetFs(afero.Fs) + IsSet(string) bool + AllKeys() []string + SetEnvKeyTransformer(string, func(data string) interface{}) + GetStringSliceE(string) ([]string, error) + GetStringMapE(string) (map[string]interface{}, error) + GetStringMapStringE(string) (map[string]string, error) + GetStringMapStringSliceE(string) (map[string][]string, error) + GetSizeInBytesE(string) (uint, error) +} + +type notImplMethodsImpl struct{} + +func (n *notImplMethodsImpl) SetFs(afero.Fs) { + n.logErrorNotImplemented("SetFs") +} + +func (n *notImplMethodsImpl) IsSet(string) bool { + n.logErrorNotImplemented("IsSet") + return false +} + +func (n *notImplMethodsImpl) AllKeys() []string { + n.logErrorNotImplemented("AllKeys") + return nil +} + +func (n *notImplMethodsImpl) SetEnvKeyTransformer(string, func(data string) interface{}) { + n.logErrorNotImplemented("SetEnvKeyTransformer") +} + +func (n *notImplMethodsImpl) GetStringSliceE(string) ([]string, error) { + return nil, n.logErrorNotImplemented("GetStringSliceE") +} + +func (n *notImplMethodsImpl) GetStringMapE(string) (map[string]interface{}, error) { + return nil, n.logErrorNotImplemented("GetStringMapE") +} + +func (n *notImplMethodsImpl) GetStringMapStringE(string) (map[string]string, error) { + return nil, n.logErrorNotImplemented("GetStringMapStringE") +} + +func (n *notImplMethodsImpl) GetStringMapStringSliceE(string) (map[string][]string, error) { + return nil, n.logErrorNotImplemented("GetStringMapStringSliceE") +} + +func (n *notImplMethodsImpl) GetSizeInBytesE(string) (uint, error) { + return 0, n.logErrorNotImplemented("GetSizeInBytesE") +} + +func (n *notImplMethodsImpl) logErrorNotImplemented(method string) error { + err := fmt.Errorf("not implemented: %s", method) + log.Error(err) + return err +} diff --git a/pkg/config/nodetreemodel/read_config_file.go b/pkg/config/nodetreemodel/read_config_file.go new file mode 100644 index 0000000000000..3cc574472453c --- /dev/null +++ b/pkg/config/nodetreemodel/read_config_file.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "io" + "os" + + "gopkg.in/yaml.v2" +) + +func (c *ntmConfig) getConfigFile() string { + if c.configFile == "" { + return "datadog.yaml" + } + return c.configFile +} + +// ReadInConfig wraps Viper for concurrent access +func (c *ntmConfig) ReadInConfig() error { + c.Lock() + defer c.Unlock() + // ReadInConfig reset configuration with the main config file + err := c.readInConfig() + if err != nil { + return err + } + + // Read extra config files + // TODO: handle c.extraConfigFilePaths, read and merge files + return nil +} + +func (c *ntmConfig) readInConfig() error { + filename := c.getConfigFile() + content, err := os.ReadFile(filename) + if err != nil { + return err + } + root, err := c.readConfigurationContent(content) + if err != nil { + return err + } + c.root = root + return nil +} + +// ReadConfig wraps Viper for concurrent access +func (c *ntmConfig) ReadConfig(in io.Reader) error { + c.Lock() + defer c.Unlock() + content, err := io.ReadAll(in) + if err != nil { + return err + } + root, err := c.readConfigurationContent(content) + if err != nil { + return err + } + c.root = root + return nil +} + +func (c *ntmConfig) readConfigurationContent(content []byte) (Node, error) { + var obj map[string]interface{} + if err := yaml.Unmarshal(content, &obj); err != nil { + return nil, err + } + root, err := NewNode(obj) + if err != nil { + return nil, err + } + return root, nil +} diff --git a/pkg/config/nodetreemodel/read_config_file_test.go b/pkg/config/nodetreemodel/read_config_file_test.go new file mode 100644 index 0000000000000..65a88192254a1 --- /dev/null +++ b/pkg/config/nodetreemodel/read_config_file_test.go @@ -0,0 +1,55 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +var confYaml = ` +network_devices: + snmp_traps: + enabled: true + port: 1234 + community_strings: ["a","b","c"] + users: + - user: alice + authKey: hunter2 + authProtocol: MD5 + privKey: pswd + privProtocol: AE5 + - user: bob + authKey: "123456" + authProtocol: MD5 + privKey: secret + privProtocol: AE5 + bind_host: ok + stop_timeout: 4 + namespace: abc +` + +func TestReadConfigAndGetValues(t *testing.T) { + cfg := NewConfig("datadog", "DD", nil) + err := cfg.ReadConfig(strings.NewReader(confYaml)) + if err != nil { + panic(err) + } + + enabled := cfg.GetBool("network_devices.snmp_traps.enabled") + port := cfg.GetInt("network_devices.snmp_traps.port") + bindHost := cfg.GetString("network_devices.snmp_traps.bind_host") + stopTimeout := cfg.GetInt("network_devices.snmp_traps.stop_timeout") + namespace := cfg.GetString("network_devices.snmp_traps.namespace") + + assert.Equal(t, enabled, true) + assert.Equal(t, port, 1234) + assert.Equal(t, bindHost, "ok") + assert.Equal(t, stopTimeout, 4) + assert.Equal(t, namespace, "abc") +} diff --git a/pkg/config/nodetreemodel/reflection_node.go b/pkg/config/nodetreemodel/reflection_node.go new file mode 100644 index 0000000000000..41abc6d3c9a3a --- /dev/null +++ b/pkg/config/nodetreemodel/reflection_node.go @@ -0,0 +1,139 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "fmt" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +var ( + // error when a caller tries to construct a node from reflect.Value, this is a logic error, calling code should + // not be reflection based, but should be working with "native" go types that come from parsing json, yaml, etc + errReflectValue = fmt.Errorf("refusing to construct node from reflect.Value") + errUnknownConversion = fmt.Errorf("no conversion found") +) + +// asReflectionNode returns a node using reflection: should only show up in test code +// The reason is that data produced by parsing json, yaml, etc should always made up +// of "plain" go-lang types (maps, slices, scalars). Some unit tests assign structs directly +// to the state of the config, which then require reflection to properly handle +func asReflectionNode(v interface{}) (Node, error) { + if _, ok := v.(reflect.Value); ok { + return nil, errReflectValue + } + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Struct { + return &structNodeImpl{val: rv}, nil + } else if rv.Kind() == reflect.Slice { + elems := make([]interface{}, 0, rv.Len()) + for i := 0; i < rv.Len(); i++ { + node, err := NewNode(rv.Index(i).Interface()) + if err != nil { + return nil, err + } + elems = append(elems, node) + } + return newArrayNodeImpl(elems) + } else if rv.Kind() == reflect.Map { + res := make(map[string]interface{}, rv.Len()) + mapkeys := rv.MapKeys() + for _, mk := range mapkeys { + kstr := "" + if key, ok := mk.Interface().(string); ok { + kstr = key + } else { + kstr = fmt.Sprintf("%s", mk.Interface()) + } + res[kstr] = rv.MapIndex(mk).Interface() + } + return newMapNodeImpl(res) + } + return nil, errUnknownConversion +} + +type structNodeImpl struct { + // val must be a struct + val reflect.Value +} + +// GetChild returns the child node at the given case-insensitive key, or an error if not found +func (n *structNodeImpl) GetChild(key string) (Node, error) { + findex := findFieldMatch(n.val, key) + if findex == -1 { + return nil, ErrNotFound + } + inner := n.val.Field(findex) + if inner.Kind() == reflect.Interface { + inner = inner.Elem() + } + return NewNode(inner.Interface()) +} + +// ChildrenKeys returns the list of keys of the children of the given node, if it is a map +func (n *structNodeImpl) ChildrenKeys() ([]string, error) { + structType := n.val.Type() + keys := make([]string, 0, n.val.NumField()) + for i := 0; i < structType.NumField(); i++ { + f := structType.Field(i) + ch, _ := utf8.DecodeRuneInString(f.Name) + if unicode.IsLower(ch) { + continue + } + fieldKey, _ := fieldNameToKey(f) + keys = append(keys, fieldKey) + } + return keys, nil +} + +type specifierSet map[string]struct{} + +// fieldNameToKey returns the lower-cased field name, for case insensitive comparisons, +// with struct tag rename applied, as well as the set of specifiers from struct tags +// struct tags are handled in order of yaml, then json, then mapstructure +func fieldNameToKey(field reflect.StructField) (string, specifierSet) { + name := field.Name + + tagtext := "" + if val := field.Tag.Get("yaml"); val != "" { + tagtext = val + } else if val := field.Tag.Get("json"); val != "" { + tagtext = val + } else if val := field.Tag.Get("mapstructure"); val != "" { + tagtext = val + } + + // skip any additional specifiers such as ",omitempty" or ",squash" + // TODO: support multiple specifiers + var specifiers map[string]struct{} + if commaPos := strings.IndexRune(tagtext, ','); commaPos != -1 { + specifiers = make(map[string]struct{}) + val := tagtext[:commaPos] + specifiers[tagtext[commaPos+1:]] = struct{}{} + if val != "" { + name = val + } + } else if tagtext != "" { + name = tagtext + } + return strings.ToLower(name), specifiers +} + +func findFieldMatch(val reflect.Value, key string) int { + // case-insensitive match for struct names + key = strings.ToLower(key) + schema := val.Type() + for i := 0; i < schema.NumField(); i++ { + fieldKey, _ := fieldNameToKey(schema.Field(i)) + if key == fieldKey { + return i + } + } + return -1 +} diff --git a/pkg/config/nodetreemodel/reflection_node_test.go b/pkg/config/nodetreemodel/reflection_node_test.go new file mode 100644 index 0000000000000..5a6b69d286fca --- /dev/null +++ b/pkg/config/nodetreemodel/reflection_node_test.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package nodetreemodel + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type Object struct { + Name string + Num int +} + +func TestNewReflectionNode(t *testing.T) { + n, err := NewNode(Object{ + Name: "test", + Num: 7, + }) + assert.NoError(t, err) + + keys, err := n.ChildrenKeys() + assert.NoError(t, err) + assert.Equal(t, keys, []string{"name", "num"}) + + first, err := n.GetChild("name") + assert.NoError(t, err) + + firstLeaf := first.(LeafNode) + str, err := firstLeaf.GetString() + assert.NoError(t, err) + assert.Equal(t, str, "test") + + second, err := n.GetChild("num") + assert.NoError(t, err) + + secondLeaf := second.(LeafNode) + num, err := secondLeaf.GetInt() + assert.NoError(t, err) + assert.Equal(t, num, 7) +} diff --git a/pkg/config/structure/go.mod b/pkg/config/structure/go.mod index eac4f43ab3750..2b9f64d2de73e 100644 --- a/pkg/config/structure/go.mod +++ b/pkg/config/structure/go.mod @@ -33,6 +33,7 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/config/mock v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.9.0 ) @@ -40,7 +41,6 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/pkg/config/setup v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/pkg/util/executable v0.56.0-rc.3 // indirect diff --git a/pkg/config/structure/unmarshal.go b/pkg/config/structure/unmarshal.go index c336a8bbc91d2..e59e59f8850c7 100644 --- a/pkg/config/structure/unmarshal.go +++ b/pkg/config/structure/unmarshal.go @@ -9,13 +9,12 @@ package structure import ( "fmt" "reflect" - "slices" - "strconv" "strings" "unicode" "unicode/utf8" "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/DataDog/datadog-agent/pkg/config/nodetreemodel" ) // features allowed for handling edge-cases @@ -38,9 +37,6 @@ var ConvertEmptyStringToNil UnmarshalKeyOption = func(fs *featureSet) { fs.convertEmptyStrNil = true } -// error for when a key is not found -var errNotFound = fmt.Errorf("not found") - // UnmarshalKey retrieves data from the config at the given key and deserializes it // to be stored on the target struct. It is implemented entirely using reflection, and // does not depend upon details of the data model of the config. @@ -55,7 +51,7 @@ func UnmarshalKey(cfg model.Reader, key string, target interface{}, opts ...Unma if rawval == nil { return nil } - source, err := newNode(reflect.ValueOf(rawval)) + source, err := nodetreemodel.NewNode(rawval) if err != nil { return err } @@ -69,7 +65,7 @@ func UnmarshalKey(cfg model.Reader, key string, target interface{}, opts ...Unma case reflect.Struct: return copyStruct(outValue, source, fs) case reflect.Slice: - if arr, ok := source.(arrayNode); ok { + if arr, ok := source.(nodetreemodel.ArrayNode); ok { return copyList(outValue, arr, fs) } if isEmptyString(source) { @@ -84,238 +80,6 @@ func UnmarshalKey(cfg model.Reader, key string, target interface{}, opts ...Unma } } -// leafNode represents a leaf with a scalar value - -type leafNode interface { - GetBool() (bool, error) - GetInt() (int, error) - GetFloat() (float64, error) - GetString() (string, error) -} - -type leafNodeImpl struct { - // val must be a scalar kind - val reflect.Value -} - -var _ leafNode = (*leafNodeImpl)(nil) -var _ node = (*leafNodeImpl)(nil) - -// arrayNode represents a node with an ordered array of children - -type arrayNode interface { - Size() int - Index(int) (node, error) -} - -type arrayNodeImpl struct { - // val must be a Slice with Len() and Index() - val reflect.Value -} - -var _ arrayNode = (*arrayNodeImpl)(nil) -var _ node = (*arrayNodeImpl)(nil) - -// node represents an arbitrary node of the tree - -type node interface { - GetChild(string) (node, error) - ChildrenKeys() ([]string, error) -} - -type innerNodeImpl struct { - // val must be a struct - val reflect.Value -} - -type innerMapNodeImpl struct { - // val must be a map[string]interface{} - val reflect.Value - // remapCase maps each lower-case key to the original case. This - // enables GetChild to retrieve values using case-insensitive keys - remapCase map[string]string -} - -var _ node = (*innerNodeImpl)(nil) -var _ node = (*innerMapNodeImpl)(nil) - -// all nodes, leaf, inner, and array nodes, each act as nodes -func newNode(v reflect.Value) (node, error) { - if v.Kind() == reflect.Struct { - return &innerNodeImpl{val: v}, nil - } else if v.Kind() == reflect.Map { - return &innerMapNodeImpl{val: v, remapCase: makeRemapCase(v)}, nil - } else if v.Kind() == reflect.Slice { - return &arrayNodeImpl{val: v}, nil - } else if isScalarKind(v) { - return &leafNodeImpl{val: v}, nil - } - return nil, fmt.Errorf("could not create node from: %v of type %T and kind %v", v, v, v.Kind()) -} - -// GetChild returns the child node at the given case-insensitive key, or an error if not found -func (n *innerNodeImpl) GetChild(key string) (node, error) { - findex := findFieldMatch(n.val, key) - if findex == -1 { - return nil, errNotFound - } - inner := n.val.Field(findex) - if inner.Kind() == reflect.Interface { - inner = inner.Elem() - } - return newNode(inner) -} - -// ChildrenKeys returns the list of keys of the children of the given node, if it is a map -func (n *innerNodeImpl) ChildrenKeys() ([]string, error) { - structType := n.val.Type() - keys := make([]string, 0, n.val.NumField()) - for i := 0; i < structType.NumField(); i++ { - f := structType.Field(i) - ch, _ := utf8.DecodeRuneInString(f.Name) - if unicode.IsLower(ch) { - continue - } - fieldKey, _ := fieldNameToKey(f) - keys = append(keys, fieldKey) - } - return keys, nil -} - -// GetChild returns the child node at the given case-insensitive key, or an error if not found -func (n *innerMapNodeImpl) GetChild(key string) (node, error) { - mkey := n.remapCase[strings.ToLower(key)] - inner := n.val.MapIndex(reflect.ValueOf(mkey)) - if !inner.IsValid() { - return nil, errNotFound - } - if inner.Kind() == reflect.Interface { - inner = inner.Elem() - } - return newNode(inner) -} - -// ChildrenKeys returns the list of keys of the children of the given node, if it is a map -func (n *innerMapNodeImpl) ChildrenKeys() ([]string, error) { - mapkeys := n.val.MapKeys() - keys := make([]string, 0, len(mapkeys)) - for _, kv := range mapkeys { - if kstr, ok := kv.Interface().(string); ok { - keys = append(keys, kstr) - } else { - return nil, fmt.Errorf("map node has invalid non-string key: %v", kv) - } - } - // map keys are iterated non-deterministically, sort them - slices.Sort(keys) - return keys, nil -} - -// GetChild returns an error because array node does not have children accessible by name -func (n *arrayNodeImpl) GetChild(string) (node, error) { - return nil, fmt.Errorf("arrayNodeImpl.GetChild not implemented") -} - -// ChildrenKeys returns an error because array node does not have children accessible by name -func (n *arrayNodeImpl) ChildrenKeys() ([]string, error) { - return nil, fmt.Errorf("arrayNodeImpl.ChildrenKeys not implemented") -} - -// Size returns number of children in the list -func (n *arrayNodeImpl) Size() int { - return n.val.Len() -} - -// Index returns the kth element of the list -func (n *arrayNodeImpl) Index(k int) (node, error) { - // arrayNodeImpl assumes val is an Array with Len() and Index() - elem := n.val.Index(k) - if elem.Kind() == reflect.Interface { - elem = elem.Elem() - } - return newNode(elem) -} - -// GetChild returns an error because a leaf has no children -func (n *leafNodeImpl) GetChild(key string) (node, error) { - return nil, fmt.Errorf("can't GetChild(%s) of a leaf node", key) -} - -// ChildrenKeys returns an error because a leaf has no children -func (n *leafNodeImpl) ChildrenKeys() ([]string, error) { - return nil, fmt.Errorf("can't get ChildrenKeys of a leaf node") -} - -// GetBool returns the scalar as a bool, or an error otherwise -func (n *leafNodeImpl) GetBool() (bool, error) { - if n.val.Kind() == reflect.Bool { - return n.val.Bool(), nil - } else if n.val.Kind() == reflect.Int { - return n.val.Int() != 0, nil - } else if n.val.Kind() == reflect.String { - return convertToBool(n.val.String()) - } - return false, newConversionError(n.val, "bool") -} - -// GetInt returns the scalar as a int, or an error otherwise -func (n *leafNodeImpl) GetInt() (int, error) { - switch n.val.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return int(n.val.Int()), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return int(n.val.Uint()), nil - case reflect.Float32, reflect.Float64: - return int(n.val.Float()), nil - } - return 0, newConversionError(n.val, "int") -} - -// GetFloat returns the scalar as a float64, or an error otherwise -func (n *leafNodeImpl) GetFloat() (float64, error) { - switch n.val.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return float64(n.val.Int()), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return float64(n.val.Uint()), nil - case reflect.Float32, reflect.Float64: - return float64(n.val.Float()), nil - } - return 0, newConversionError(n.val, "float") -} - -// GetString returns the scalar as a string, or an error otherwise -func (n *leafNodeImpl) GetString() (string, error) { - switch n.val.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - stringVal := strconv.FormatInt(n.val.Int(), 10) - return stringVal, nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - stringVal := strconv.FormatUint(n.val.Uint(), 10) - return stringVal, nil - case reflect.Float32: - stringVal := strconv.FormatFloat(n.val.Float(), 'f', -1, 32) - return stringVal, nil - case reflect.Float64: - stringVal := strconv.FormatFloat(n.val.Float(), 'f', -1, 64) - return stringVal, nil - case reflect.String: - return n.val.String(), nil - } - return "", newConversionError(n.val, "string") -} - -// convert a string to a bool using standard yaml constants -func convertToBool(text string) (bool, error) { - lower := strings.ToLower(text) - if lower == "y" || lower == "yes" || lower == "on" || lower == "true" || lower == "1" { - return true, nil - } else if lower == "n" || lower == "no" || lower == "off" || lower == "false" || lower == "0" { - return false, nil - } - return false, newConversionError(reflect.ValueOf(text), "bool") -} - type specifierSet map[string]struct{} // fieldNameToKey returns the lower-cased field name, for case insensitive comparisons, @@ -349,7 +113,7 @@ func fieldNameToKey(field reflect.StructField) (string, specifierSet) { return strings.ToLower(name), specifiers } -func copyStruct(target reflect.Value, source node, fs *featureSet) error { +func copyStruct(target reflect.Value, source nodetreemodel.Node, fs *featureSet) error { targetType := target.Type() for i := 0; i < targetType.NumField(); i++ { f := targetType.Field(i) @@ -369,7 +133,7 @@ func copyStruct(target reflect.Value, source node, fs *featureSet) error { continue } child, err := source.GetChild(fieldKey) - if err == errNotFound { + if err == nodetreemodel.ErrNotFound { continue } if err != nil { @@ -383,7 +147,7 @@ func copyStruct(target reflect.Value, source node, fs *featureSet) error { return nil } -func copyMap(target reflect.Value, source node, _ *featureSet) error { +func copyMap(target reflect.Value, source nodetreemodel.Node, _ *featureSet) error { // TODO: Should handle maps with more complex types in a future PR ktype := reflect.TypeOf("") vtype := reflect.TypeOf("") @@ -402,7 +166,7 @@ func copyMap(target reflect.Value, source node, _ *featureSet) error { if child == nil { continue } - if scalar, ok := child.(leafNode); ok { + if scalar, ok := child.(nodetreemodel.LeafNode); ok { if mval, err := scalar.GetString(); err == nil { results.SetMapIndex(reflect.ValueOf(mkey), reflect.ValueOf(mval)) } else { @@ -414,7 +178,7 @@ func copyMap(target reflect.Value, source node, _ *featureSet) error { return nil } -func copyLeaf(target reflect.Value, source leafNode, _ *featureSet) error { +func copyLeaf(target reflect.Value, source nodetreemodel.LeafNode, _ *featureSet) error { if source == nil { return fmt.Errorf("source value is not a scalar") } @@ -458,7 +222,7 @@ func copyLeaf(target reflect.Value, source leafNode, _ *featureSet) error { return fmt.Errorf("unsupported scalar type %v", target.Kind()) } -func copyList(target reflect.Value, source arrayNode, fs *featureSet) error { +func copyList(target reflect.Value, source nodetreemodel.ArrayNode, fs *featureSet) error { if source == nil { return fmt.Errorf("source value is not a list") } @@ -483,14 +247,14 @@ func copyList(target reflect.Value, source arrayNode, fs *featureSet) error { return nil } -func copyAny(target reflect.Value, source node, fs *featureSet) error { +func copyAny(target reflect.Value, source nodetreemodel.Node, fs *featureSet) error { if target.Kind() == reflect.Pointer { allocPtr := reflect.New(target.Type().Elem()) target.Set(allocPtr) target = allocPtr.Elem() } if isScalarKind(target) { - if leaf, ok := source.(leafNode); ok { + if leaf, ok := source.(nodetreemodel.LeafNode); ok { return copyLeaf(target, leaf, fs) } return fmt.Errorf("can't copy into target: scalar required, but source is not a leaf") @@ -499,7 +263,7 @@ func copyAny(target reflect.Value, source node, fs *featureSet) error { } else if target.Kind() == reflect.Struct { return copyStruct(target, source, fs) } else if target.Kind() == reflect.Slice { - if arr, ok := source.(arrayNode); ok { + if arr, ok := source.(nodetreemodel.ArrayNode); ok { return copyList(target, arr, fs) } return fmt.Errorf("can't copy into target: []T required, but source is not an array") @@ -509,8 +273,8 @@ func copyAny(target reflect.Value, source node, fs *featureSet) error { return fmt.Errorf("unknown value to copy: %v", target.Type()) } -func isEmptyString(source node) bool { - if leaf, ok := source.(leafNode); ok { +func isEmptyString(source nodetreemodel.Node) bool { + if leaf, ok := source.(nodetreemodel.LeafNode); ok { if str, err := leaf.GetString(); err == nil { return str == "" } @@ -522,36 +286,3 @@ func isScalarKind(v reflect.Value) bool { k := v.Kind() return (k >= reflect.Bool && k <= reflect.Float64) || k == reflect.String } - -func makeRemapCase(v reflect.Value) map[string]string { - remap := make(map[string]string) - iter := v.MapRange() - for iter.Next() { - mkey := "" - switch k := iter.Key().Interface().(type) { - case string: - mkey = k - default: - mkey = fmt.Sprintf("%s", k) - } - remap[strings.ToLower(mkey)] = mkey - } - return remap -} - -func findFieldMatch(val reflect.Value, key string) int { - // case-insensitive match for struct names - key = strings.ToLower(key) - schema := val.Type() - for i := 0; i < schema.NumField(); i++ { - fieldKey, _ := fieldNameToKey(schema.Field(i)) - if key == fieldKey { - return i - } - } - return -1 -} - -func newConversionError(v reflect.Value, expectType string) error { - return fmt.Errorf("could not convert to %s: %v of type %T and Kind %v", expectType, v, v, v.Kind()) -} diff --git a/pkg/config/structure/unmarshal_test.go b/pkg/config/structure/unmarshal_test.go index e268281ddda84..61f7811ba7057 100644 --- a/pkg/config/structure/unmarshal_test.go +++ b/pkg/config/structure/unmarshal_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/pkg/config/mock" + "github.com/DataDog/datadog-agent/pkg/config/nodetreemodel" "github.com/stretchr/testify/assert" ) @@ -1258,13 +1259,13 @@ service: } func TestMapGetChildNotFound(t *testing.T) { - m := map[string]string{"a": "apple", "b": "banana"} - n, err := newNode(reflect.ValueOf(m)) + m := map[string]interface{}{"a": "apple", "b": "banana"} + n, err := nodetreemodel.NewNode(m) assert.NoError(t, err) val, err := n.GetChild("a") assert.NoError(t, err) - str, err := val.(leafNode).GetString() + str, err := val.(nodetreemodel.LeafNode).GetString() assert.NoError(t, err) assert.Equal(t, str, "apple")