storage/s3: add legacy flag for backward compatibility with ListObject (

peak#407) * storage/s3: add legacy flag for backward compatibility with ListObject Signed-off-by: Paul Greenberg <greenpau@outlook.com> * add use-v1-api flag
lizzzcai · Apr 5, 2022 · ca37216 · ca37216
1 parent 60b1ac1
commit ca37216
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 - Added `--ignore-glacier-warnings` flag to `cp`, `mv` and `select` commands. ([#346](https://github.com/peak/s5cmd/issues/346))
 - Added `--force-glacier-transfer` flag to `select` command. ([#404](https://github.com/peak/s5cmd/pull/404))
 - Added AWS Single Sign-On (SSO) profiles support ([#385](https://github.com/peak/s5cmd/issues/385))
+- Added `--use-v1-api` flag to force using S3 ListObjects API instead of ListObjectsV2 API. ([#405](https://github.com/peak/s5cmd/issues/405)
 
 #### Bugfixes
 - Fixed a bug about precedence of region detection, which auto region detection would always override region defined in environment or profile. ([#325](https://github.com/peak/s5cmd/issues/325))

diff --git a/README.md b/README.md
@@ -43,6 +43,7 @@ storage services and local filesystems.
 - Google Cloud Storage (and any other S3 API compatible service) support
 - Structured logging for querying command outputs
 - Shell auto-completion
+- S3 ListObjects API backward compatibility
 
 ## Installation
 
@@ -375,6 +376,14 @@ however, those copy operations will not be performed. It is displaying what
 Note that `--dry-run` can be used with any operation that has a side effect, i.e.,
 cp, mv, rm, mb ...
 
+### S3 ListObjects API Backward Compatibility
+
+The `--use-v1-api` flag will force using S3 ListObjects API instead of ListObjectsV2 API.
+
+```
+s5cmd --use-v1-api ls
+```
+
 ### Specifying credentials
 
 `s5cmd` uses official AWS SDK to access S3. SDK requires credentials to sign

diff --git a/command/app.go b/command/app.go
@@ -69,6 +69,10 @@ var app = &cli.App{
 			Name:  "no-sign-request",
 			Usage: "do not sign requests: credentials will not be loaded if --no-sign-request is provided",
 		},
+		&cli.BoolFlag{
+			Name:  "use-v1-api",
+			Usage: "enable backward compatibility with ListObjects API and disable ListObjectsV2 API",
+		},
 	},
 	Before: func(c *cli.Context) error {
 		retryCount := c.Int("retry-count")
@@ -139,6 +143,7 @@ func NewStorageOpts(c *cli.Context) storage.Options {
 		NoVerifySSL:   c.Bool("no-verify-ssl"),
 		DryRun:        c.Bool("dry-run"),
 		NoSignRequest: c.Bool("no-sign-request"),
+		APIv1Enabled:  c.Bool("use-v1-api"),
 	}
 }
 

diff --git a/storage/s3.go b/storage/s3.go
@@ -58,6 +58,7 @@ type S3 struct {
 	uploader    s3manageriface.UploaderAPI
 	endpointURL urlpkg.URL
 	dryRun      bool
+	v1Enabled   bool
 }
 
 func parseEndpoint(endpoint string) (urlpkg.URL, error) {
@@ -95,6 +96,7 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) {
 		uploader:    s3manager.NewUploader(awsSession),
 		endpointURL: endpointURL,
 		dryRun:      opts.DryRun,
+		v1Enabled:   opts.APIv1Enabled,
 	}, nil
 }
 
@@ -128,6 +130,9 @@ func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object {
 	if isGoogleEndpoint(s.endpointURL) {
 		return s.listObjects(ctx, url)
 	}
+	if s.v1Enabled {
+		return s.listObjects(ctx, url)
+	}
 	return s.listObjectsV2(ctx, url)
 }
 

diff --git a/storage/storage.go b/storage/storage.go
@@ -53,6 +53,7 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro
 		NoVerifySSL:   opts.NoVerifySSL,
 		DryRun:        opts.DryRun,
 		NoSignRequest: opts.NoSignRequest,
+		APIv1Enabled:  opts.APIv1Enabled,
 		bucket:        url.Bucket,
 		region:        opts.region,
 	}
@@ -73,6 +74,7 @@ type Options struct {
 	NoVerifySSL   bool
 	DryRun        bool
 	NoSignRequest bool
+	APIv1Enabled  bool
 	bucket        string
 	region        string
 }