Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

command/cp: add cross-region transfer support #194

Merged
merged 28 commits into from
Oct 22, 2020
Merged
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
c9ddb46
command/{app, cp}, storage/{s3, s3_test}: added cross-region transfer…
fbarotov Jul 16, 2020
4244867
command/{}, storage/{}, e2e/cp_test: batch run for cross-region transfer
fbarotov Jul 17, 2020
3891977
Merge branch 'master' into transfer
fbarotov Jul 17, 2020
7d53d22
merge and few fixes
fbarotov Jul 22, 2020
ed32853
Merge branch 'master' of https://github.com/peak/s5cmd into transfer
fbarotov Jul 24, 2020
b50bed9
command/, storage/: updates based on pr review
fbarotov Jul 24, 2020
7d1d18f
storage/{s3, s3_test}: updates based on pr review
fbarotov Jul 25, 2020
4ad98e0
command/, readme: updates based on pr review
fbarotov Jul 28, 2020
540b11c
Merge branch 'master' of https://github.com/peak/s5cmd into transfer
fbarotov Jul 28, 2020
4ab4450
command/{app, cp, run}: updates based on pr review
fbarotov Jul 28, 2020
ab3dfb4
command/: updates based on pr review
fbarotov Jul 29, 2020
cb13391
storage/{s3, s3_test}: updates based on pr review
fbarotov Jul 29, 2020
377ed32
command/{app, cp, options}, storage/: updates based on pr review
fbarotov Jul 29, 2020
1b19b3a
storage/s3_test: update based on feedback: more? testcase
fbarotov Aug 4, 2020
053de4f
storage/s3_test: fixed an error from the last commit
fbarotov Aug 4, 2020
4a7730a
storage/ : updates based on pr review
fbarotov Aug 8, 2020
c77c2ce
command/: `infer` source and destination bucket regions
fbarotov Aug 10, 2020
cf9d41e
command/{cp, run}: removed unnecessary flags
fbarotov Aug 10, 2020
2611610
command/ : updates based on pr review
fbarotov Aug 11, 2020
2692fe7
changelog, command/ : some roll-backs
fbarotov Aug 11, 2020
f957e57
merged with remote master
fbarotov Aug 18, 2020
cd0f119
merge with upstream master
fbarotov Sep 30, 2020
4e54be4
storage/storage: removed region from options
fbarotov Oct 1, 2020
080ce0e
storage/s3: updates based on pr review
fbarotov Oct 2, 2020
868569c
storage/s3: updates based on pr review
fbarotov Oct 5, 2020
56a0963
storage/s3: updates based on pr review
fbarotov Oct 17, 2020
339ad77
storage/s3: updates based on pr review
fbarotov Oct 20, 2020
cc3115d
changelog: updates based on pr review
fbarotov Oct 21, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
command/{app, cp, options}, storage/: updates based on pr review
fbarotov committed Jul 29, 2020
commit 377ed32bcb1bb4f9dd1e9a71e74e09bb82b349fc
33 changes: 0 additions & 33 deletions command/app.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@ import (
"context"
"fmt"

"github.com/peak/s5cmd/storage"
cmpinstall "github.com/posener/complete/cmd/install"
"github.com/urfave/cli/v2"

@@ -114,35 +113,3 @@ func Main(ctx context.Context, args []string) error {

return app.RunContext(ctx, args)
}

// s3opts returns new S3Options object by extracting
// its fields from the provided context. Region is
// taken as (default) source-region.
func s3opts(c *cli.Context) storage.S3Options {
region := c.String("source-region")
if region == "" {
region = c.String("default-source-region")
}
return storage.S3Options{
MaxRetries: c.Int("retry-count"),
Endpoint: c.String("endpoint-url"),
NoVerifySSL: c.Bool("no-verify-ssl"),
Region: region,
}
}

// dstS3opts returns new S3Options object by extracting
// its fields from the provided context. Region is
// taken as (default) region, i.e., destination region.
func dstS3opts(c *cli.Context) storage.S3Options {
dstRegion := c.String("region")
if dstRegion == "" {
dstRegion = c.String("default-region")
}
return storage.S3Options{
MaxRetries: c.Int("retry-count"),
Endpoint: c.String("endpoint-url"),
NoVerifySSL: c.Bool("no-verify-ssl"),
Region: dstRegion,
}
}
9 changes: 4 additions & 5 deletions command/cp.go
Original file line number Diff line number Diff line change
@@ -477,11 +477,6 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
}

func (c Copy) doCopy(ctx context.Context, srcurl *url.URL, dsturl *url.URL) error {
srcClient, err := storage.NewClient(srcurl, c.srcS3opts)
if err != nil {
return err
}

dstClient, err := storage.NewClient(dsturl, c.dstS3opts)
if err != nil {
return err
@@ -508,6 +503,10 @@ func (c Copy) doCopy(ctx context.Context, srcurl *url.URL, dsturl *url.URL) erro
}

if c.deleteSource {
srcClient, err := storage.NewClient(srcurl, c.srcS3opts)
if err != nil {
return err
}
if err := srcClient.Delete(ctx, srcurl); err != nil {
return err
}
38 changes: 38 additions & 0 deletions command/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package command

import (
"github.com/peak/s5cmd/storage"
"github.com/urfave/cli/v2"
)

// s3opts returns new S3Options object by extracting
// its fields from the provided context. Region is
// taken as (default) source-region.
func s3opts(c *cli.Context) storage.S3Options {
region := c.String("source-region")
if region == "" {
region = c.String("default-source-region")
}
return storage.S3Options{
MaxRetries: c.Int("retry-count"),
Endpoint: c.String("endpoint-url"),
NoVerifySSL: c.Bool("no-verify-ssl"),
Region: region,
}
}

// dstS3opts returns new S3Options object by extracting
// its fields from the provided context. Region is
// taken as (default) region, i.e., destination region.
func dstS3opts(c *cli.Context) storage.S3Options {
dstRegion := c.String("region")
if dstRegion == "" {
dstRegion = c.String("default-region")
}
return storage.S3Options{
MaxRetries: c.Int("retry-count"),
Endpoint: c.String("endpoint-url"),
NoVerifySSL: c.Bool("no-verify-ssl"),
Region: dstRegion,
}
}
16 changes: 0 additions & 16 deletions storage/s3.go
Original file line number Diff line number Diff line change
@@ -668,22 +668,6 @@ func (s *s3Session) newSession(opts S3Options) (*session.Session, error) {
return sess, nil
}

// NumOfSessions returns number of sessions currently active.
func NumOfSessions() int {
s := sessions
s.Lock()
defer s.Unlock()
return len(s.sessions)
}

// ClearSessions clears all existing sessions.
func ClearSessions() {
s := sessions
s.Lock()
defer s.Unlock()
s.sessions = map[S3Options]*session.Session{}
}

// customRetryer wraps the SDK's built in DefaultRetryer adding additional
// error codes. Such as, retry for S3 InternalError code.
type customRetryer struct {
87 changes: 0 additions & 87 deletions storage/s3_test.go
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@ import (
"math/rand"
"net/http"
urlpkg "net/url"
"os"
"reflect"
"strings"
"testing"
@@ -79,31 +78,6 @@ func TestNewSessionPathStyle(t *testing.T) {
}
}

func TestNewSessionWithRegionSetViaEnv(t *testing.T) {
opts := S3Options{
Region: "",
}

const expectedRegion = "us-west-2"

if NumOfSessions() > 0 {
ClearSessions()
}

os.Setenv("AWS_REGION", expectedRegion)
defer os.Unsetenv("AWS_REGION")

sess, err := sessions.newSession(opts)
if err != nil {
t.Fatal(err)
}

got := aws.StringValue(sess.Config.Region)
if got != expectedRegion {
t.Fatalf("expected %v, got %v", expectedRegion, got)
}
}

func TestS3ListSuccess(t *testing.T) {
url, err := url.New("s3://bucket/key")
if err != nil {
@@ -417,67 +391,6 @@ func TestS3Retry(t *testing.T) {
}
}

func TestNumOfSessions(t *testing.T) {
ClearSessions()

testcases := []struct {
name string
sourceRegion string
destinationRegion string

expectedSessions int
}{
{
name: "different source-region and region",
sourceRegion: "cn-north-1",
destinationRegion: "eu-central-1",

expectedSessions: 2,
},
{
name: "same source-region and region",
sourceRegion: "eu-west-1",
destinationRegion: "eu-west-1",

expectedSessions: 1,
},
{
name: "source-region and region not set",
expectedSessions: 1,
},
}
const expectedTotalNumOfSessions = 4
for _, tc := range testcases {
tc := tc

t.Run(tc.name, func(t *testing.T) {
defer ClearSessions()

for _, r := range []string{tc.sourceRegion, tc.destinationRegion} {
_, err := NewS3Storage(S3Options{
Region: r,
})
if err != nil {
t.Error(err)
}
}

assert.Equal(t, NumOfSessions(), tc.expectedSessions)
})
}
for _, tc := range testcases {
for _, r := range []string{tc.sourceRegion, tc.destinationRegion} {
_, err := NewS3Storage(S3Options{
Region: r,
})
if err != nil {
t.Error(err)
}
}
}
assert.Equal(t, NumOfSessions(), expectedTotalNumOfSessions)
}

// Credit to aws-sdk-go
func val(i interface{}, s string) interface{} {
v, err := awsutil.ValuesAtPath(i, s)