Skip to content

Commit

Permalink
Azure Blob: update SDK and add access tier support
Browse files Browse the repository at this point in the history
  • Loading branch information
@drakkan
drakkan committed Oct 30, 2020
1 parent ebb18fa commit fcfdd63
Show file tree
Hide file tree
Showing 10 changed files with 116 additions and 72 deletions.
4 changes: 4 additions & 0 deletions cmd/portable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ var (
portableAzAccountName string
portableAzAccountKey string
portableAzEndpoint string
portableAzAccessTier string
portableAzSASURL string
portableAzKeyPrefix string
portableAzULPartSize int
Expand Down Expand Up @@ -164,6 +165,7 @@ Please take a look at the usage below to customize the serving parameters`,
AccountName: portableAzAccountName,
AccountKey: portableAzAccountKey,
Endpoint: portableAzEndpoint,
AccessTier: portableAzAccessTier,
SASURL: portableAzSASURL,
KeyPrefix: portableAzKeyPrefix,
UseEmulator: portableAzUseEmulator,
Expand Down Expand Up @@ -272,6 +274,8 @@ HTTPS`)
portableCmd.Flags().StringVar(&portableAzSASURL, "az-sas-url", "", `Shared access signature URL`)
portableCmd.Flags().StringVar(&portableAzEndpoint, "az-endpoint", "", `Leave empty to use the default:
"blob.core.windows.net"`)
portableCmd.Flags().StringVar(&portableAzAccessTier, "az-access-tier", "", `Leave empty to use the default
container setting`)
portableCmd.Flags().StringVar(&portableAzKeyPrefix, "az-key-prefix", "", `Allows to restrict access to the
virtual folder identified by this
prefix and its contents`)
Expand Down
40 changes: 22 additions & 18 deletions examples/rest-api-cli/sftpgo_api_cli
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@ class SFTPGoApiRequests:
s3_key_prefix='', gcs_bucket='', gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='',
gcs_automatic_credentials='automatic', denied_login_methods=[], virtual_folders=[],
denied_extensions=[], allowed_extensions=[], s3_upload_part_size=0, s3_upload_concurrency=0,
max_upload_file_size=0, denied_protocols=[], az_container="", az_account_name="", az_account_key="",
az_sas_url="", az_endpoint="", az_upload_part_size=0, az_upload_concurrency=0, az_key_prefix="",
az_use_emulator=False):
max_upload_file_size=0, denied_protocols=[], az_container='', az_account_name='', az_account_key='',
az_sas_url='', az_endpoint='', az_upload_part_size=0, az_upload_concurrency=0, az_key_prefix='',
az_use_emulator=False, az_access_tier=''):
user = {'id':user_id, 'username':username, 'uid':uid, 'gid':gid,
'max_sessions':max_sessions, 'quota_size':quota_size, 'quota_files':quota_files,
'upload_bandwidth':upload_bandwidth, 'download_bandwidth':download_bandwidth,
Expand All @@ -111,7 +111,7 @@ class SFTPGoApiRequests:
gcs_automatic_credentials, s3_upload_part_size, s3_upload_concurrency,
az_container, az_account_name, az_account_key, az_sas_url,
az_endpoint, az_upload_part_size, az_upload_concurrency, az_key_prefix,
az_use_emulator)})
az_use_emulator, az_access_tier)})
return user

def buildVirtualFolders(self, vfolders):
Expand Down Expand Up @@ -235,7 +235,7 @@ class SFTPGoApiRequests:
s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
gcs_credentials_file, gcs_automatic_credentials, s3_upload_part_size, s3_upload_concurrency,
az_container, az_account_name, az_account_key, az_sas_url, az_endpoint, az_upload_part_size,
az_upload_concurrency, az_key_prefix, az_use_emulator):
az_upload_concurrency, az_key_prefix, az_use_emulator, az_access_tier):
fs_config = {'provider':0}
if fs_provider == 'S3':
s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
Expand All @@ -254,10 +254,10 @@ class SFTPGoApiRequests:
'automatic_credentials':0})
fs_config.update({'provider':2, 'gcsconfig':gcsconfig})
elif fs_provider == "AzureBlob":
azureconfig = {"container":az_container, "account_name":az_account_name, "account_key":az_account_key,
"sas_url":az_sas_url, "endpoint":az_endpoint, "upload_part_size":az_upload_part_size,
"upload_concurrency":az_upload_concurrency, "key_prefix":az_key_prefix, "use_emulator":
az_use_emulator}
azureconfig = {'container':az_container, 'account_name':az_account_name, 'account_key':az_account_key,
'sas_url':az_sas_url, 'endpoint':az_endpoint, 'upload_part_size':az_upload_part_size,
'upload_concurrency':az_upload_concurrency, 'key_prefix':az_key_prefix, 'use_emulator':
az_use_emulator, 'access_tier':az_access_tier}
fs_config.update({'provider':3, 'azblobconfig':azureconfig})
return fs_config

Expand All @@ -277,16 +277,16 @@ class SFTPGoApiRequests:
gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='', gcs_automatic_credentials='automatic',
denied_login_methods=[], virtual_folders=[], denied_extensions=[], allowed_extensions=[],
s3_upload_part_size=0, s3_upload_concurrency=0, max_upload_file_size=0, denied_protocols=[], az_container="",
az_account_name="", az_account_key="", az_sas_url="", az_endpoint="", az_upload_part_size=0,
az_upload_concurrency=0, az_key_prefix="", az_use_emulator=False):
az_account_name='', az_account_key='', az_sas_url='', az_endpoint='', az_upload_part_size=0,
az_upload_concurrency=0, az_key_prefix='', az_use_emulator=False, az_access_tier=''):
u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
gcs_credentials_file, gcs_automatic_credentials, denied_login_methods, virtual_folders, denied_extensions,
allowed_extensions, s3_upload_part_size, s3_upload_concurrency, max_upload_file_size, denied_protocols,
az_container, az_account_name, az_account_key, az_sas_url, az_endpoint, az_upload_part_size,
az_upload_concurrency, az_key_prefix, az_use_emulator)
az_upload_concurrency, az_key_prefix, az_use_emulator, az_access_tier)
r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)

Expand All @@ -297,17 +297,17 @@ class SFTPGoApiRequests:
s3_key_prefix='', gcs_bucket='', gcs_key_prefix='', gcs_storage_class='', gcs_credentials_file='',
gcs_automatic_credentials='automatic', denied_login_methods=[], virtual_folders=[], denied_extensions=[],
allowed_extensions=[], s3_upload_part_size=0, s3_upload_concurrency=0, max_upload_file_size=0,
denied_protocols=[], disconnect=0, az_container="",
az_account_name="", az_account_key="", az_sas_url="", az_endpoint="", az_upload_part_size=0,
az_upload_concurrency=0, az_key_prefix="", az_use_emulator=False):
denied_protocols=[], disconnect=0, az_container='', az_account_name='', az_account_key='', az_sas_url='',
az_endpoint='', az_upload_part_size=0, az_upload_concurrency=0, az_key_prefix='', az_use_emulator=False,
az_access_tier=''):
u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix, gcs_bucket, gcs_key_prefix, gcs_storage_class,
gcs_credentials_file, gcs_automatic_credentials, denied_login_methods, virtual_folders, denied_extensions,
allowed_extensions, s3_upload_part_size, s3_upload_concurrency, max_upload_file_size, denied_protocols,
az_container, az_account_name, az_account_key, az_sas_url, az_endpoint, az_upload_part_size,
az_upload_concurrency, az_key_prefix, az_use_emulator)
az_upload_concurrency, az_key_prefix, az_use_emulator, az_access_tier)
r = requests.put(urlparse.urljoin(self.userPath, 'user/' + str(user_id)), params={'disconnect':disconnect},
json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)
Expand Down Expand Up @@ -642,6 +642,8 @@ def addCommonUserArguments(parser):
parser.add_argument('--az-account-key', type=str, default='', help='Default: %(default)s')
parser.add_argument('--az-sas-url', type=str, default='', help='Shared access signature URL. Default: %(default)s')
parser.add_argument('--az-endpoint', type=str, default='', help='Default: %(default)s')
parser.add_argument('--az-access-tier', type=str, default='', choices=['', 'Hot', 'Cool', 'Archive'],
help='Default: %(default)s')
parser.add_argument('--az-upload-part-size', type=int, default=0, help='The buffer size for multipart uploads (MB). ' +
'Zero means the default (1 MB). Default: %(default)s')
parser.add_argument('--az-upload-concurrency', type=int, default=0, help='How many parts are uploaded in parallel. ' +
Expand Down Expand Up @@ -805,7 +807,8 @@ if __name__ == '__main__':
args.denied_login_methods, args.virtual_folders, args.denied_extensions, args.allowed_extensions,
args.s3_upload_part_size, args.s3_upload_concurrency, args.max_upload_file_size, args.denied_protocols,
args.az_container, args.az_account_name, args.az_account_key, args.az_sas_url, args.az_endpoint,
args.az_upload_part_size, args.az_upload_concurrency, args.az_key_prefix, args.az_use_emulator)
args.az_upload_part_size, args.az_upload_concurrency, args.az_key_prefix, args.az_use_emulator,
args.az_access_tier)
elif args.command == 'update-user':
api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
Expand All @@ -817,7 +820,8 @@ if __name__ == '__main__':
args.virtual_folders, args.denied_extensions, args.allowed_extensions, args.s3_upload_part_size,
args.s3_upload_concurrency, args.max_upload_file_size, args.denied_protocols, args.disconnect,
args.az_container, args.az_account_name, args.az_account_key, args.az_sas_url, args.az_endpoint,
args.az_upload_part_size, args.az_upload_concurrency, args.az_key_prefix, args.az_use_emulator)
args.az_upload_part_size, args.az_upload_concurrency, args.az_key_prefix, args.az_use_emulator,
args.az_access_tier)
elif args.command == 'delete-user':
api.deleteUser(args.id)
elif args.command == 'get-users':
Expand Down
26 changes: 14 additions & 12 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ module github.com/drakkan/sftpgo
go 1.14

require (
cloud.google.com/go v0.69.1 // indirect
cloud.google.com/go v0.70.0 // indirect
cloud.google.com/go/storage v1.12.0
github.com/Azure/azure-storage-blob-go v0.10.0
github.com/Azure/azure-storage-blob-go v0.11.0
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b
github.com/aws/aws-sdk-go v1.35.9
github.com/aws/aws-sdk-go v1.35.19
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
github.com/fclairamb/ftpserverlib v0.8.1-0.20201007044523-7c6c9f869560
github.com/fsnotify/fsnotify v1.4.9 // indirect
Expand All @@ -21,11 +22,11 @@ require (
github.com/lib/pq v1.8.0
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-sqlite3 v1.14.4
github.com/miekg/dns v1.1.34 // indirect
github.com/miekg/dns v1.1.35 // indirect
github.com/mitchellh/mapstructure v1.3.3 // indirect
github.com/otiai10/copy v1.2.0
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/pires/go-proxyproto v0.2.0
github.com/pires/go-proxyproto v0.3.0
github.com/pkg/sftp v1.12.1-0.20201002132022-fcaa492add82
github.com/prometheus/client_golang v1.8.0
github.com/rs/cors v1.7.1-0.20200626170627-8b4a00bd362b
Expand All @@ -41,19 +42,20 @@ require (
go.etcd.io/bbolt v1.3.5
go.uber.org/automaxprocs v1.3.0
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0
golang.org/x/sys v0.0.0-20201018121011-98379d014ca7
golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 // indirect
google.golang.org/api v0.33.0
golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418
golang.org/x/text v0.3.4 // indirect
golang.org/x/tools v0.0.0-20201030174515-8da1a626af69 // indirect
google.golang.org/api v0.34.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20201015140912-32ed001d685c // indirect
google.golang.org/grpc v1.33.0 // indirect
google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3 // indirect
google.golang.org/grpc v1.33.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)

replace (
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20200730125632-b21eac28818c
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20201017144935-4e8324213ac3
golang.org/x/net => github.com/drakkan/net v0.0.0-20201017144739-aa7ab65e4c0d
golang.org/x/net => github.com/drakkan/net v0.0.0-20201030182451-5fdd610a97c6
)
Loading

0 comments on commit fcfdd63

Please sign in to comment.