Changed session helper to handle optional region

Move the parameter store to aws_helper Add utility functions to handle s3 paths + tests Replace all TerragruntOptions structure initialization by a call to options.NewTerragruntOptionsForTest Reorganize the main loop to handle variables definition properly Add functionalities to import_files (copy_and_rename, import_into_modules, initialization from a remote source) Fix a logging problem on -all operations (error were logged as notification) Disabled the redundant get on backend init Implemented a cache mechanism to optimize the file importation from S3
coveooss · Jul 7, 2017 · 570e48f · 570e48f
1 parent 8547de6
commit 570e48f
Show file tree

Hide file tree

Showing 20 changed files with 932 additions and 551 deletions.
diff --git a/aws_helper/config.go b/aws_helper/config.go
@@ -8,13 +8,17 @@ import (
 
 // Returns an AWS session object for the given region, ensuring that the credentials are available
 func CreateAwsSession(awsRegion, awsProfile string) (*session.Session, error) {
-	session, err := session.NewSessionWithOptions(session.Options{
-		Config:            aws.Config{Region: aws.String(awsRegion)},
+	options := session.Options{
 		Profile:           awsProfile,
 		SharedConfigState: session.SharedConfigEnable,
-	})
+	}
+	if awsRegion != "" {
+		options.Config = aws.Config{Region: aws.String(awsRegion)}
+	}
+	session, err := session.NewSessionWithOptions(options)
+
 	if err != nil {
-		return nil, errors.WithStackTraceAndPrefix(err, "Error intializing session")
+		return nil, errors.WithStackTraceAndPrefix(err, "Error initializing session")
 	}
 
 	_, err = session.Config.Credentials.Get()

diff --git a/util/parameter_store.go → aws_helper/parameter_store.go b/util/parameter_store.go → aws_helper/parameter_store.go
@@ -1,16 +1,18 @@
-package util
+package aws_helper
 
 import (
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ssm"
 )
 
 // GetSSMParameter returns the value from the parameters store
-func GetSSMParameter(parameterName string) (string, error) {
+func GetSSMParameter(parameterName, region string) (string, error) {
+	session, err := CreateAwsSession(region, "")
+	if err != nil {
+		return "", err
+	}
+
+	svc := ssm.New(session)
 	withDecryption := true
-	svc := ssm.New(session.Must(session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-	})))
 	result, err := svc.GetParameter(&ssm.GetParameterInput{
 		Name:           &parameterName,
 		WithDecryption: &withDecryption,

diff --git a/util/parameter_store_test.go → aws_helper/parameter_store_test.go b/util/parameter_store_test.go → aws_helper/parameter_store_test.go
@@ -1,4 +1,4 @@
-package util
+package aws_helper
 
 import "testing"
 
@@ -16,7 +16,7 @@ func TestGetSSMParameter(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetSSMParameter(tt.args.parameterName)
+			got, err := GetSSMParameter(tt.args.parameterName, "")
 			if (err != nil) != tt.wantErr {
 				t.Errorf("GetSSMParameter(%s) error = %v, wantErr %v", tt.args.parameterName, err, tt.wantErr)
 				return

diff --git a/aws_helper/s3.go b/aws_helper/s3.go
@@ -0,0 +1,179 @@
+package aws_helper
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// ConvertS3Path returns an S3 compatible path
+func ConvertS3Path(path string) (string, error) {
+	if !strings.HasPrefix(path, "s3://") {
+		return path, nil
+	}
+
+	parts := strings.Split(path[5:], "/")
+
+	session, err := CreateAwsSession("us-east-1", "")
+	if err != nil {
+		return formatS3Path(parts[0], "", parts[1:]...), err
+	}
+	svc := s3.New(session)
+
+	answer, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: aws.String(parts[0])})
+	if err != nil {
+		return formatS3Path(parts[0], "", parts[1:]...), err
+	}
+
+	region := ""
+	if answer.LocationConstraint != nil {
+		region = "-" + *answer.LocationConstraint
+	}
+
+	return formatS3Path(parts[0], region, parts[1:]...), nil
+}
+
+func formatS3Path(bucket, region string, parts ...string) string {
+	key := strings.Join(parts, "/")
+	if key != "" {
+		key = "/" + key
+	}
+	return fmt.Sprintf("%s.s3%s.amazonaws.com%s", bucket, region, key)
+}
+
+// GetBucketObjectInfoFromURL retrieve the components of the bucket (name, key, region) from an URL
+func GetBucketObjectInfoFromURL(url string) (*BucketInfo, error) {
+	if s3Patterns == nil {
+		s3Patterns = []*regexp.Regexp{
+			regexp.MustCompile(`^https?://(?P<bucket>[^/\.]+?).s3.amazonaws.com(?:/(?P<key>.*))?$`),
+			regexp.MustCompile(`^https?://(?P<bucket>[^/\.]+?).s3-(?P<region>.*?).amazonaws.com(?:/(?P<key>.*))?$`),
+			regexp.MustCompile(`^https?://s3.amazonaws.com/(?P<bucket>[^/\.]+?)(?:/(?P<key>.*))?$`),
+			regexp.MustCompile(`^https?://s3-(?P<region>.*?).amazonaws.com/(?P<bucket>[^/\.]+?)(?:/(?P<key>.*))?$`),
+		}
+	}
+
+	convertedURL, _ := ConvertS3Path(url)
+	if !strings.HasPrefix(convertedURL, "http") {
+		convertedURL = "https://" + convertedURL
+	}
+
+	for _, pattern := range s3Patterns {
+		matches := pattern.FindStringSubmatch(convertedURL)
+		if matches != nil {
+			result := &BucketInfo{}
+			for i, part := range pattern.SubexpNames() {
+				switch part {
+				case "bucket":
+					result.BucketName = matches[i]
+				case "key":
+					result.Key = matches[i]
+				case "region":
+					result.Region = matches[i]
+				}
+			}
+
+			return result, nil
+		}
+	}
+	return nil, fmt.Errorf("Non valid bucket url %s", url)
+}
+
+// BucketInfo represents the basic information relative to an S3 URL
+type BucketInfo struct {
+	BucketName string
+	Region     string
+	Key        string
+}
+
+type bucketStatus struct {
+	BucketInfo
+	Etag         string
+	Version      string
+	LastModified time.Time
+}
+
+var s3Patterns []*regexp.Regexp
+
+// SaveS3Status save the current state of the S3 bucket folder in the directory
+func SaveS3Status(url, folder string) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s %v", url, err)
+		}
+	}()
+	bucketInfo, err := GetBucketObjectInfoFromURL(url)
+	if err != nil {
+		return err
+	}
+
+	if !strings.HasSuffix(bucketInfo.Key, "/") {
+		bucketInfo.Key += "/"
+	}
+
+	status, err := getS3Status(*bucketInfo)
+	if err != nil {
+		return
+	}
+
+	jsonString, err := json.Marshal(status)
+	if err != nil {
+		return
+	}
+	err = ioutil.WriteFile(filepath.Join(folder, cacheFile), jsonString, 0644)
+	return nil
+}
+
+// CheckS3Status compares the saved status with the current version of the bucket folder
+// returns true if the objects has not changed
+func CheckS3Status(folder string) bool {
+	content, err := ioutil.ReadFile(filepath.Join(folder, cacheFile))
+	if err != nil {
+		return false
+	}
+
+	var status bucketStatus
+	err = json.Unmarshal(content, &status)
+	if err != nil {
+		return false
+	}
+
+	s3Status, err := getS3Status(status.BucketInfo)
+	if err != nil {
+		return false
+	}
+
+	return reflect.DeepEqual(status, *s3Status)
+}
+
+const cacheFile = ".terragrunt.cache"
+
+func getS3Status(info BucketInfo) (*bucketStatus, error) {
+	session, err := CreateAwsSession(info.Region, "")
+	if err != nil {
+		return nil, err
+	}
+	svc := s3.New(session)
+
+	answer, err := svc.HeadObject(&s3.HeadObjectInput{
+		Bucket: aws.String(info.BucketName),
+		Key:    aws.String(info.Key),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &bucketStatus{
+		BucketInfo:   info,
+		Etag:         *answer.ETag,
+		Version:      *answer.VersionId,
+		LastModified: *answer.LastModified,
+	}, nil
+}
diff --git a/aws_helper/s3_test.go b/aws_helper/s3_test.go
@@ -0,0 +1,68 @@
+package aws_helper
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestConvertS3Path(t *testing.T) {
+	type args struct {
+		path string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{"Non S3 path", args{"/folder/test"}, "/folder/test", false},
+		{"Non available S3", args{"s3://bucket/test"}, "bucket.s3.amazonaws.com/test", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ConvertS3Path(tt.args.path)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ConvertS3Path() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ConvertS3Path() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetBucketObjectInfoFromURL(t *testing.T) {
+	tests := []struct {
+		url     string
+		want    *BucketInfo
+		wantErr bool
+	}{
+		{"/bucket", nil, true},
+		{"s3://bucket", &BucketInfo{"bucket", "", ""}, false},
+		{"s3://bucket/key", &BucketInfo{"bucket", "", "key"}, false},
+		{"bucket.s3.amazonaws.com", &BucketInfo{"bucket", "", ""}, false},
+		{"bucket.s3-us-east-1.amazonaws.com/key/file", &BucketInfo{"bucket", "us-east-1", "key/file"}, false},
+		{"s3.amazonaws.com/bucket", &BucketInfo{"bucket", "", ""}, false},
+		{"s3-us-east-1.amazonaws.com/bucket/key/file", &BucketInfo{"bucket", "us-east-1", "key/file"}, false},
+		{"http://bucket.s3.amazonaws.com", &BucketInfo{"bucket", "", ""}, false},
+		{"http://bucket.s3.amazonaws.com/key", &BucketInfo{"bucket", "", "key"}, false},
+		{"https://bucket.s3.amazonaws.com", &BucketInfo{"bucket", "", ""}, false},
+		{"https://bucket.s3.amazonaws.com/key", &BucketInfo{"bucket", "", "key"}, false},
+		{"https://bucket.s3-us-east-1.amazonaws.com", &BucketInfo{"bucket", "us-east-1", ""}, false},
+		{"https://bucket.s3-us-west-2.amazonaws.com/key", &BucketInfo{"bucket", "us-west-2", "key"}, false},
+		{"https://s3-us-east-2.amazonaws.com/bucket", &BucketInfo{"bucket", "us-east-2", ""}, false},
+	}
+	for _, tt := range tests {
+		t.Run(" ("+tt.url+")", func(t *testing.T) {
+			got, err := GetBucketObjectInfoFromURL(tt.url)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Error = %v, Expected %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/cli/args_test.go b/cli/args_test.go
@@ -2,14 +2,15 @@ package cli
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
 	"github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/errors"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/util"
 	"github.com/stretchr/testify/assert"
-	"os"
-	"path/filepath"
-	"testing"
 )
 
 func TestParseTerragruntOptionsFromArgs(t *testing.T) {
@@ -207,15 +208,8 @@ func TestParseEnvironmentVariables(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		var mockOptions = options.TerragruntOptions{
-			TerragruntConfigPath: "test-env-mock",
-			NonInteractive:       true,
-			Env:                  map[string]string{},
-			Variables:            options.VariableList{},
-			Logger:               util.CreateLogger(""),
-		}
-
-		parseEnvironmentVariables(&mockOptions, testCase.environmentVariables)
+		var mockOptions = options.NewTerragruntOptionsForTest("test-env-mock")
+		parseEnvironmentVariables(mockOptions, testCase.environmentVariables)
 		assert.Equal(t, testCase.expectedVariables, mockOptions.Env)
 	}
 }