Skip to content
This repository has been archived by the owner on Apr 19, 2023. It is now read-only.

Added Exponential Backoff for most all (most) operations. #431

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added Exponential Backoff for most all (most) operations. #431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions drive/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,14 @@ type DeleteArgs struct {
Recursive bool
}

func (self *Drive) Delete(args DeleteArgs) error {
func (self *Drive) Delete(args DeleteArgs, try int) error {
f, err := self.service.Files.Get(args.Id).Fields("name", "mimeType").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Delete(args, try)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand All @@ -23,16 +28,26 @@ func (self *Drive) Delete(args DeleteArgs) error {

err = self.service.Files.Delete(args.Id).Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Delete(args, try)
}
return fmt.Errorf("Failed to delete file: %s", err)
}

fmt.Fprintf(args.Out, "Deleted '%s'\n", f.Name)
return nil
}

func (self *Drive) deleteFile(fileId string) error {
func (self *Drive) deleteFile(fileId string, try int) error {
err := self.service.Files.Delete(fileId).Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.deleteFile(fileId, try)
}
return fmt.Errorf("Failed to delete file: %s", err)
}
return nil
Expand Down
38 changes: 26 additions & 12 deletions drive/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,18 @@ type DownloadArgs struct {
Timeout time.Duration
}

func (self *Drive) Download(args DownloadArgs) error {
func (self *Drive) Download(args DownloadArgs, try int) error {
if args.Recursive {
return self.downloadRecursive(args)
return self.downloadRecursive(args, 1)
}

f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Download(args, try)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand All @@ -42,7 +47,7 @@ func (self *Drive) Download(args DownloadArgs) error {
return fmt.Errorf("'%s' is a google document and must be exported, see the export command", f.Name)
}

bytes, rate, err := self.downloadBinary(f, args)
bytes, rate, err := self.downloadBinary(f, args, 1)
if err != nil {
return err
}
Expand All @@ -52,7 +57,7 @@ func (self *Drive) Download(args DownloadArgs) error {
}

if args.Delete {
err = self.deleteFile(args.Id)
err = self.deleteFile(args.Id, 1)
if err != nil {
return fmt.Errorf("Failed to delete file: %s", err)
}
Expand All @@ -79,7 +84,7 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
query: args.Query,
fields: []googleapi.Field{"nextPageToken", "files(id,name,mimeType,size,md5Checksum)"},
}
files, err := self.listAllFiles(listArgs)
files, err := self.listAllFiles(listArgs, 1)
if err != nil {
return fmt.Errorf("Failed to list files: %s", err)
}
Expand All @@ -96,7 +101,7 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
if isDir(f) && args.Recursive {
err = self.downloadDirectory(f, downloadArgs)
} else if isBinary(f) {
_, _, err = self.downloadBinary(f, downloadArgs)
_, _, err = self.downloadBinary(f, downloadArgs, 1)
}

if err != nil {
Expand All @@ -107,29 +112,38 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
return nil
}

func (self *Drive) downloadRecursive(args DownloadArgs) error {
func (self *Drive) downloadRecursive(args DownloadArgs, try int) error {
f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.downloadRecursive(args, try)
}
return fmt.Errorf("Failed to get file: %s", err)
}

if isDir(f) {
return self.downloadDirectory(f, args)
} else if isBinary(f) {
_, _, err = self.downloadBinary(f, args)
_, _, err = self.downloadBinary(f, args, 1)
return err
}

return nil
}

func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int64, error) {
func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs, try int) (int64, int64, error) {
// Get timeout reader wrapper and context
timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout)

res, err := self.service.Files.Get(f.Id).Context(ctx).Download()
if err != nil {
if isTimeoutError(err) {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.downloadBinary(f, args, try)
} else if isTimeoutError(err) {
return 0, 0, fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout)
}
return 0, 0, fmt.Errorf("Failed to download file: %s", err)
Expand Down Expand Up @@ -228,7 +242,7 @@ func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) erro
query: fmt.Sprintf("'%s' in parents", parent.Id),
fields: []googleapi.Field{"nextPageToken", "files(id,name)"},
}
files, err := self.listAllFiles(listArgs)
files, err := self.listAllFiles(listArgs, 1)
if err != nil {
return fmt.Errorf("Failed listing files: %s", err)
}
Expand All @@ -242,7 +256,7 @@ func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) erro
newArgs.Id = f.Id
newArgs.Stdout = false

err = self.downloadRecursive(newArgs)
err = self.downloadRecursive(newArgs, 1)
if err != nil {
return err
}
Expand Down
12 changes: 11 additions & 1 deletion drive/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,14 @@ type ExportArgs struct {
Force bool
}

func (self *Drive) Export(args ExportArgs) error {
func (self *Drive) Export(args ExportArgs, try int) error {
f, err := self.service.Files.Get(args.Id).Fields("name", "mimeType").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Export(args, try)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand All @@ -43,6 +48,11 @@ func (self *Drive) Export(args ExportArgs) error {

res, err := self.service.Files.Export(args.Id, exportMime).Download()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Export(args, try)
}
return fmt.Errorf("Failed to download file: %s", err)
}

Expand Down
2 changes: 1 addition & 1 deletion drive/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func (self *Drive) Import(args ImportArgs) error {
Path: args.Path,
Parents: args.Parents,
Mime: toMimes[0],
})
}, 1)
if err != nil {
return err
}
Expand Down
7 changes: 6 additions & 1 deletion drive/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,14 @@ type FileInfoArgs struct {
SizeInBytes bool
}

func (self *Drive) Info(args FileInfoArgs) error {
func (self *Drive) Info(args FileInfoArgs, try int) error {
f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "createdTime", "modifiedTime", "md5Checksum", "mimeType", "parents", "shared", "description", "webContentLink", "webViewLink").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.Info(args, try)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand Down
14 changes: 10 additions & 4 deletions drive/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func (self *Drive) List(args ListFilesArgs) (err error) {
sortOrder: args.SortOrder,
maxFiles: args.MaxFiles,
}
files, err := self.listAllFiles(listArgs)
files, err := self.listAllFiles(listArgs, 1)
if err != nil {
return fmt.Errorf("Failed to list files: %s", err)
}
Expand Down Expand Up @@ -62,7 +62,7 @@ type listAllFilesArgs struct {
maxFiles int64
}

func (self *Drive) listAllFiles(args listAllFilesArgs) ([]*drive.File, error) {
func (self *Drive) listAllFiles(args listAllFilesArgs, try int) ([]*drive.File, error) {
var files []*drive.File

var pageSize int64
Expand All @@ -85,8 +85,14 @@ func (self *Drive) listAllFiles(args listAllFilesArgs) ([]*drive.File, error) {
return nil
})

if err != nil && err != controlledStop {
return nil, err
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.listAllFiles(args, try)
} else if err != controlledStop {
return nil, err
}
}

if args.maxFiles > 0 {
Expand Down
10 changes: 8 additions & 2 deletions drive/mkdir.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ type MkdirArgs struct {
}

func (self *Drive) Mkdir(args MkdirArgs) error {
f, err := self.mkdir(args)
f, err := self.mkdir(args, 1)
if err != nil {
return err
}
fmt.Fprintf(args.Out, "Directory %s created\n", f.Id)
return nil
}

func (self *Drive) mkdir(args MkdirArgs) (*drive.File, error) {
func (self *Drive) mkdir(args MkdirArgs, try int) (*drive.File, error) {
dstFile := &drive.File{
Name: args.Name,
Description: args.Description,
Expand All @@ -37,6 +37,12 @@ func (self *Drive) mkdir(args MkdirArgs) (*drive.File, error) {
// Create directory
f, err := self.service.Files.Create(dstFile).Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.mkdir(args, try)
}

return nil, fmt.Errorf("Failed to create directory: %s", err)
}

Expand Down
9 changes: 7 additions & 2 deletions drive/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,14 @@ func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp File
}, nil
}

func (self *Drive) isSyncFile(id string) (bool, error) {
func (self *Drive) isSyncFile(id string, try int) (bool, error) {
f, err := self.service.Files.Get(id).Fields("appProperties").Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.isSyncFile(id, try)
}
return false, fmt.Errorf("Failed to get file: %s", err)
}

Expand Down Expand Up @@ -159,7 +164,7 @@ func (self *Drive) prepareRemoteFiles(rootDir *drive.File, sortOrder string) ([]
fields: []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType,size,modifiedTime)"},
sortOrder: sortOrder,
}
files, err := self.listAllFiles(listArgs)
files, err := self.listAllFiles(listArgs, 1)
if err != nil {
return nil, fmt.Errorf("Failed listing files: %s", err)
}
Expand Down
9 changes: 7 additions & 2 deletions drive/sync_download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func (self *Drive) DownloadSync(args DownloadSyncArgs) error {
started := time.Now()

// Get remote root dir
rootDir, err := self.getSyncRoot(args.RootId)
rootDir, err := self.getSyncRoot(args.RootId, 1)
if err != nil {
return err
}
Expand Down Expand Up @@ -83,10 +83,15 @@ func (self *Drive) DownloadSync(args DownloadSyncArgs) error {
return nil
}

func (self *Drive) getSyncRoot(rootId string) (*drive.File, error) {
func (self *Drive) getSyncRoot(rootId string, try int) (*drive.File, error) {
fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
f, err := self.service.Files.Get(rootId).Fields(fields...).Do()
if err != nil {
if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
exponentialBackoffSleep(try)
try++
return self.getSyncRoot(rootId, try)
}
return nil, fmt.Errorf("Failed to find root dir: %s", err)
}

Expand Down
4 changes: 2 additions & 2 deletions drive/sync_list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func (self *Drive) ListSync(args ListSyncArgs) error {
query: "appProperties has {key='syncRoot' and value='true'}",
fields: []googleapi.Field{"nextPageToken", "files(id,name,mimeType,createdTime)"},
}
files, err := self.listAllFiles(listArgs)
files, err := self.listAllFiles(listArgs, 1)
if err != nil {
return err
}
Expand All @@ -37,7 +37,7 @@ type ListRecursiveSyncArgs struct {
}

func (self *Drive) ListRecursiveSync(args ListRecursiveSyncArgs) error {
rootDir, err := self.getSyncRoot(args.RootId)
rootDir, err := self.getSyncRoot(args.RootId, 1)
if err != nil {
return err
}
Expand Down
Loading