diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e2aaf79..2fad068ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,14 @@ ## not released yet +#### Features +- Added `--profile` flag to allow users to specify a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). ([#353](https://github.com/peak/s5cmd/issues/353)) +- Added `--credentials-file` flag to allow users to specify path for the AWS credentials file instead of using the [default location](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where). + #### Bugfixes - Fixed a bug where (`--stat`) prints unnecessarily when used with help and version commands ([#452](https://github.com/peak/s5cmd/issues/452)) - Changed cp error message to be more precise. "given object not found" error message now will also include absolute path of the file. ([#463](https://github.com/peak/s5cmd/pull/463)) - #### Improvements - Disable AWS SDK logger if log level is not "trace" diff --git a/README.md b/README.md index ffd343727..f0f2a35ee 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,7 @@ s5cmd --use-list-objects-v1 ls s3://bucket/ `s5cmd` uses official AWS SDK to access S3. SDK requires credentials to sign requests to AWS. Credentials can be provided in a variety of ways: - +- Command line options `--profile` to use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html), `--credentials-file` flag to use the specified credentials file, and `--no-sign-request` to send requests anonymously - Environment variables - AWS credentials file, including profile selection via `AWS_PROFILE` environment variable diff --git a/command/app.go b/command/app.go index 14c970eb2..23f4588e6 100644 --- a/command/app.go +++ b/command/app.go @@ -81,6 +81,14 @@ var app = &cli.App{ Name: "request-payer", Usage: "who pays for request (access requester pays buckets)", }, + &cli.StringFlag{ + Name: "profile", + Usage: "use the specified profile from the credentials file", + }, + &cli.StringFlag{ + Name: "credentials-file", + Usage: "use the specified credentials file instead of the default credentials file", + }, }, Before: func(c *cli.Context) error { retryCount := c.Int("retry-count") @@ -97,6 +105,16 @@ var app = &cli.App{ printError(commandFromContext(c), c.Command.Name, err) return err } + if c.Bool("no-sign-request") && c.String("profile") != "" { + err := fmt.Errorf(`"no-sign-request" and "profile" flags cannot be used together`) + printError(commandFromContext(c), c.Command.Name, err) + return err + } + if c.Bool("no-sign-request") && c.String("credentials-file") != "" { + err := fmt.Errorf(`"no-sign-request" and "credentials-file" flags cannot be used together`) + printError(commandFromContext(c), c.Command.Name, err) + return err + } if isStat { stat.InitStat() @@ -162,6 +180,8 @@ func NewStorageOpts(c *cli.Context) storage.Options { NoVerifySSL: c.Bool("no-verify-ssl"), RequestPayer: c.String("request-payer"), UseListObjectsV1: c.Bool("use-list-objects-v1"), + Profile: c.String("profile"), + CredentialFile: c.String("credentials-file"), LogLevel: log.LevelFromString(c.String("log")), } } diff --git a/storage/s3.go b/storage/s3.go index 09dc69511..3877fd0f8 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -770,7 +770,11 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session. if opts.NoSignRequest { // do not sign requests when making service API calls - awsCfg.Credentials = credentials.AnonymousCredentials + awsCfg = awsCfg.WithCredentials(credentials.AnonymousCredentials) + } else if opts.CredentialFile != "" || opts.Profile != "" { + awsCfg = awsCfg.WithCredentials( + credentials.NewSharedCredentials(opts.CredentialFile, opts.Profile), + ) } endpointURL, err := parseEndpoint(opts.Endpoint) diff --git a/storage/s3_test.go b/storage/s3_test.go index 42de3ce25..a31f2f0ce 100644 --- a/storage/s3_test.go +++ b/storage/s3_test.go @@ -128,6 +128,88 @@ func TestNewSessionWithNoSignRequest(t *testing.T) { } } +func TestNewSessionWithProfileFromFile(t *testing.T) { + // create a temporary credentials file + file, err := os.CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + profiles := `[default] +aws_access_key_id = default_profile_key_id +aws_secret_access_key = default_profile_access_key + +[p1] +aws_access_key_id = p1_profile_key_id +aws_secret_access_key = p1_profile_access_key + +[p2] +aws_access_key_id = p2_profile_key_id +aws_secret_access_key = p2_profile_access_key` + + _, err = file.Write([]byte(profiles)) + if err != nil { + t.Fatal(err) + } + + testcases := []struct { + name string + fileName string + profileName string + expAccessKeyId string + expSecretAccessKey string + }{ + { + name: "use default profile", + fileName: file.Name(), + profileName: "", + expAccessKeyId: "default_profile_key_id", + expSecretAccessKey: "default_profile_access_key", + }, + { + name: "use a non-default profile", + fileName: file.Name(), + profileName: "p1", + expAccessKeyId: "p1_profile_key_id", + expSecretAccessKey: "p1_profile_access_key", + }, + { + + name: "use a non-existent profile", + fileName: file.Name(), + profileName: "non-existent-profile", + expAccessKeyId: "", + expSecretAccessKey: "", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + globalSessionCache.clear() + sess, err := globalSessionCache.newSession(context.Background(), Options{ + Profile: tc.profileName, + CredentialFile: tc.fileName, + }) + if err != nil { + t.Fatal(err) + } + + got, err := sess.Config.Credentials.Get() + if err != nil { + // if there should be such a profile but received an error fail, + // ignore the error otherwise. + if tc.expAccessKeyId != "" || tc.expSecretAccessKey != "" { + t.Fatal(err) + } + } + + if got.AccessKeyID != tc.expAccessKeyId || got.SecretAccessKey != tc.expSecretAccessKey { + t.Errorf("Expected credentials does not match the credential we got!\nExpected: Access Key ID: %v, Secret Access Key: %v\nGot : Access Key ID: %v, Secret Access Key: %v\n", tc.expAccessKeyId, tc.expSecretAccessKey, got.AccessKeyID, got.SecretAccessKey) + } + }) + } +} + func TestS3ListURL(t *testing.T) { url, err := url.New("s3://bucket/key") if err != nil { diff --git a/storage/storage.go b/storage/storage.go index 1b56780d6..77d063186 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -63,6 +63,8 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro NoSignRequest: opts.NoSignRequest, UseListObjectsV1: opts.UseListObjectsV1, RequestPayer: opts.RequestPayer, + Profile: opts.Profile, + CredentialFile: opts.CredentialFile, LogLevel: opts.LogLevel, bucket: url.Bucket, region: opts.region, @@ -87,6 +89,8 @@ type Options struct { UseListObjectsV1 bool LogLevel log.LogLevel RequestPayer string + Profile string + CredentialFile string bucket string region string }