confmap: Add support for nested URIs (open-telemetry#7504)

* confmap: Add support for nested URIs Addresses open-telemetry#7117. This PR adds support for nesting URIs, e.g. ``` test: "${http://example.com/?os=${env:OS}}" ``` **Breaking change:** In the case of an InvalidScheme, error `invalid uri: "g_c_s:VALUE"` will be thrown. Previously, no error was thrown (see [test case](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.74.0/confmap/resolver_test.go#L623-L625)). Other than the above, there will be no breaking changes. Although the parser now provides the ability to understand why parsing a URI fails (e.g. missing opening `${`), no errors will be thrown to preserve the previous behaviour. The old way to expand env vars (e.g. `${HOST}`) is not supported in nested URIs, as expanding the old way is done in a converter. This has been documented. * add issue # to changelog * address feedback * address feedback * address feedback * Address feedback: change expandStringURI & preserve comment * Address feedback: move recursion to findURI * Address feedback: avoid duplicate calls to findURI * Update confmap/expand.go Co-authored-by: Bogdan Drutu <lazy@splunk.com> * Address Feedback * remove unused errURILimit * update findURI documentation * fix test * Update confmap/expand.go * Address feedback: remove named return values --------- Co-authored-by: Bogdan Drutu <lazy@splunk.com>
alemrtv · Apr 15, 2023 · fe508cf · fe508cf
1 parent cc471de
commit fe508cf
Show file tree

Hide file tree

Showing 7 changed files with 702 additions and 469 deletions.
diff --git a/.chloggen/support-nested-URIs-breaking.yaml b/.chloggen/support-nested-URIs-breaking.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Using an Invalid Scheme in a URI will throw an error.
+
+# One or more tracking issues or pull requests related to the change
+issues: [7504]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/.chloggen/support-nested-URIs.yaml b/.chloggen/support-nested-URIs.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: confmap
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for nested URIs.
+
+# One or more tracking issues or pull requests related to the change
+issues: [7117]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/confmap/README.md b/confmap/README.md
@@ -37,8 +37,9 @@ that can be used by code that is oblivious to the usage of `Providers` and `Conv
 or an individual value (partial configuration) when the `configURI` is embedded into the `Conf` as a values using
 the syntax `${configURI}`.
 
-**Limitation:** when embed a `${configURI}` the uri cannot contain dollar sign ("$") character. This is to allow the
-current implementation to evolve in the future to support embedded uri within uri, e.g. `${http://my.domain.com?os=${OS}}`.
+**Limitation:** 
+- When embedding a `${configURI}` the uri cannot contain dollar sign ("$") character unless it embeds another uri.
+- The number of URIs is limited to 100.
 
 ```terminal
               Resolver                   Provider

diff --git a/confmap/expand.go b/confmap/expand.go
@@ -0,0 +1,184 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package confmap // import "go.opentelemetry.io/collector/confmap"
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// schemePattern defines the regexp pattern for scheme names.
+// Scheme name consist of a sequence of characters beginning with a letter and followed by any
+// combination of letters, digits, plus ("+"), period ("."), or hyphen ("-").
+const schemePattern = `[A-Za-z][A-Za-z0-9+.-]+`
+
+var (
+	// Need to match new line as well in the OpaqueValue, so setting the "s" flag. See https://pkg.go.dev/regexp/syntax.
+	uriRegexp = regexp.MustCompile(`(?s:^(?P<Scheme>` + schemePattern + `):(?P<OpaqueValue>.*)$)`)
+
+	errTooManyRecursiveExpansions = errors.New("too many recursive expansions")
+)
+
+func (mr *Resolver) expandValueRecursively(ctx context.Context, value any) (any, error) {
+	for i := 0; i < 100; i++ {
+		val, changed, err := mr.expandValue(ctx, value)
+		if err != nil {
+			return nil, err
+		}
+		if !changed {
+			return val, nil
+		}
+		value = val
+	}
+	return nil, errTooManyRecursiveExpansions
+}
+
+func (mr *Resolver) expandValue(ctx context.Context, value any) (any, bool, error) {
+	switch v := value.(type) {
+	case string:
+		if !strings.Contains(v, "${") || !strings.Contains(v, "}") {
+			// No URIs to expand.
+			return value, false, nil
+		}
+		// Embedded or nested URIs.
+		return mr.findAndExpandURI(ctx, v)
+	case []any:
+		nslice := make([]any, 0, len(v))
+		nchanged := false
+		for _, vint := range v {
+			val, changed, err := mr.expandValue(ctx, vint)
+			if err != nil {
+				return nil, false, err
+			}
+			nslice = append(nslice, val)
+			nchanged = nchanged || changed
+		}
+		return nslice, nchanged, nil
+	case map[string]any:
+		nmap := map[string]any{}
+		nchanged := false
+		for mk, mv := range v {
+			val, changed, err := mr.expandValue(ctx, mv)
+			if err != nil {
+				return nil, false, err
+			}
+			nmap[mk] = val
+			nchanged = nchanged || changed
+		}
+		return nmap, nchanged, nil
+	}
+	return value, false, nil
+}
+
+// findURI attempts to find the first expandable URI in input. It returns an expandable
+// URI, or an empty string if none are found.
+// Note: findURI is only called when input contains a closing bracket.
+func findURI(input string) string {
+	closeIndex := strings.Index(input, "}")
+	remaining := input[closeIndex+1:]
+	openIndex := strings.LastIndex(input[:closeIndex+1], "${")
+
+	// if there is a missing "${" or the uri does not contain ":", check the next URI.
+	if openIndex < 0 || !strings.Contains(input[openIndex:closeIndex+1], ":") {
+		// if remaining does not contain "}", there are no URIs left: stop recursion.
+		if !strings.Contains(remaining, "}") {
+			return ""
+		}
+		return findURI(remaining)
+	}
+
+	return input[openIndex : closeIndex+1]
+}
+
+// findAndExpandURI attempts to find and expand the first occurrence of an expandable URI in input. If an expandable URI is found it
+// returns the input with the URI expanded, true and nil. Otherwise, it returns the unchanged input, false and the expanding error.
+func (mr *Resolver) findAndExpandURI(ctx context.Context, input string) (any, bool, error) {
+	uri := findURI(input)
+	if uri == "" {
+		// No URI found, return.
+		return input, false, nil
+	}
+	if uri == input {
+		// If the value is a single URI, then the return value can be anything.
+		// This is the case `foo: ${file:some_extra_config.yml}`.
+		return mr.expandURI(ctx, input)
+	}
+	expanded, changed, err := mr.expandURI(ctx, uri)
+	if err != nil {
+		return input, false, err
+	}
+	repl, err := toString(uri, expanded)
+	if err != nil {
+		return input, false, err
+	}
+	return strings.ReplaceAll(input, uri, repl), changed, err
+}
+
+// toString attempts to convert input to a string.
+func toString(strURI string, input any) (string, error) {
+	// This list must be kept in sync with checkRawConfType.
+	val := reflect.ValueOf(input)
+	switch val.Kind() {
+	case reflect.String:
+		return val.String(), nil
+	case reflect.Int, reflect.Int32, reflect.Int64:
+		return strconv.FormatInt(val.Int(), 10), nil
+	case reflect.Float32, reflect.Float64:
+		return strconv.FormatFloat(val.Float(), 'f', -1, 64), nil
+	case reflect.Bool:
+		return strconv.FormatBool(val.Bool()), nil
+	default:
+		return "", fmt.Errorf("expanding %v, expected convertable to string value type, got %q(%T)", strURI, input, input)
+	}
+}
+
+func (mr *Resolver) expandURI(ctx context.Context, uri string) (any, bool, error) {
+	lURI, err := newLocation(uri[2 : len(uri)-1])
+	if err != nil {
+		return nil, false, err
+	}
+	if strings.Contains(lURI.opaqueValue, "$") {
+		return nil, false, fmt.Errorf("the uri %q contains unsupported characters ('$')", lURI.asString())
+	}
+	ret, err := mr.retrieveValue(ctx, lURI)
+	if err != nil {
+		return nil, false, err
+	}
+	mr.closers = append(mr.closers, ret.Close)
+	val, err := ret.AsRaw()
+	return val, true, err
+}
+
+type location struct {
+	scheme      string
+	opaqueValue string
+}
+
+func (c location) asString() string {
+	return c.scheme + ":" + c.opaqueValue
+}
+
+func newLocation(uri string) (location, error) {
+	submatches := uriRegexp.FindStringSubmatch(uri)
+	if len(submatches) != 3 {
+		return location{}, fmt.Errorf("invalid uri: %q", uri)
+	}
+	return location{scheme: submatches[1], opaqueValue: submatches[2]}, nil
+}