sync_s3: enable syncing to other cloud providers compatible with s3

* added '--s3host' and '--acl' options * instead of using key.set_acl, 'policy' keyword argument in set_contents_from_string was used (this approach is inspired by django-storages) * get_s3connection_kwargs added for easier overriding/extending * docs updated and extended * additionaly: found typos corrected
tommeagher · Dec 26, 2013 · 1c921fd · 1c921fd
1 parent e65b5f4
commit 1c921fd
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 8 deletions.
diff --git a/django_extensions/management/commands/sync_s3.py b/django_extensions/management/commands/sync_s3.py
@@ -10,6 +10,9 @@
 * gzip compress any CSS and Javascript files it finds and adds the appropriate
   'Content-Encoding' header.
 * set a far future 'Expires' header for optimal caching.
+* upload only media or static files.
+* use any other provider compatible with Amazon S3.
+* set other than 'public-read' ACL.
 
 Note: This script requires the Python boto library and valid Amazon Web
 Services API keys.
@@ -33,18 +36,20 @@
   --force               Skip the file mtime check to force upload of all
                         files.
   --filter-list         Override default directory and file exclusion
-                        filters. (enter as comma seperated line)
+                        filters. (enter as comma separated line)
   --renamegzip          Enables renaming of gzipped files by appending '.gz'.
                         to the original file name. This way your original
                         assets will not be replaced by the gzipped ones.
                         You can change the extension setting the
                         `SYNC_S3_RENAME_GZIP_EXT` var in your settings.py
                         file.
-  --invalidate          Invalidates the objects in CloudFront after uploaading
+  --invalidate          Invalidates the objects in CloudFront after uploading
                         stuff to s3.
   --media-only          Only MEDIA_ROOT files will be uploaded to S3.
   --static-only         Only STATIC_ROOT files will be uploaded to S3.
-
+  --s3host              Override default s3 host.
+  --acl                 Override default ACL settings ('public-read' if
+                        settings.AWS_DEFAULT_ACL is not defined).
 
 TODO:
  * Use fnmatch (or regex) to allow more complex FILTER_LIST rules.
@@ -105,6 +110,14 @@ class Command(BaseCommand):
         make_option('-d', '--dir',
                     dest='dir',
                     help="Custom static root directory to use"),
+        make_option('--s3host',
+                    dest='s3host',
+                    default=getattr(settings, 'AWS_S3_HOST', ''),
+                    help="The s3 host (enables connecting to other providers/regions)"),
+        make_option('--acl',
+                    dest='acl',
+                    default=getattr(settings, 'AWS_DEFAULT_ACL', 'public-read'),
+                    help="Enables to override default acl (public-read)."),
         make_option('--gzip',
                     action='store_true', dest='gzip', default=False,
                     help="Enables gzipping CSS and Javascript files."),
@@ -173,6 +186,8 @@ def handle(self, *args, **options):
         self.do_force = options.get('force')
         self.invalidate = options.get('invalidate')
         self.DIRECTORIES = options.get('dir')
+        self.s3host = options.get('s3host')
+        self.default_acl = options.get('acl')
         self.FILTER_LIST = getattr(settings, 'FILTER_LIST', self.FILTER_LIST)
         filter_list = options.get('filter_list')
         if filter_list:
@@ -255,11 +270,21 @@ def compress_string(self, s):
         zfile.close()
         return zbuf.getvalue()
 
+    def get_s3connection_kwargs(self):
+        """Returns connection kwargs as a dict"""
+        kwargs = {}
+        if self.s3host:
+            kwargs['host'] = self.s3host
+        return kwargs
+
     def open_s3(self):
         """
         Opens connection to S3 returning bucket and key
         """
-        conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
+        conn = boto.connect_s3(
+            self.AWS_ACCESS_KEY_ID,
+            self.AWS_SECRET_ACCESS_KEY,
+            **self.get_s3connection_kwargs())
         try:
             bucket = conn.get_bucket(self.AWS_BUCKET_NAME)
         except boto.exception.S3ResponseError:
@@ -345,8 +370,8 @@ def upload_s3(self, arg, dirname, names):
 
             try:
                 key.name = file_key
-                key.set_contents_from_string(filedata, headers, replace=True)
-                key.set_acl('public-read')
+                key.set_contents_from_string(filedata, headers, replace=True,
+                                             policy=self.default_acl)
             except boto.exception.S3CreateError as e:
                 print("Failed: %s" % e)
             except Exception as e:

diff --git a/docs/AUTHORS b/docs/AUTHORS
@@ -19,3 +19,4 @@ Tyson Clugg (tclugg) - Patched sqldiff
 Domen Kožar (iElectric) - staticfiles patch improvement
 quinox - original staticfiles patch
 Camilo Nova (camilonova)
+Wiktor Kołodziej (viciu) (http://deployed.pl)
diff --git a/docs/sync_s3.rst b/docs/sync_s3.rst
@@ -13,7 +13,8 @@ This command can optionally do the following but it is off by default:
     appropriate 'Content-Encoding' header.
   * set a far future 'Expires' header for optimal caching.
   * upload only media or static files.
-
+  * use any other provider compatible with Amazon S3.
+  * set other than 'public-read' ACL.
 
 Example Usage
 -------------
@@ -33,6 +34,10 @@ Example Usage
   # Upload only media files to S3 into the bucket 'mybucket'
   $ ./manage.py sync_s3 mybucket  --media-only  # or --static-only
 
+::
+
+  # Upload only media files to a S3 compatible provider into the bucket 'mybucket' and set private file ACLs
+  $ ./manage.py sync_s3 mybucket  --media-only  --s3host=cs.example.com --acl=private
 
 Required libraries and settings
 -------------------------------
@@ -43,8 +48,23 @@ This management command requires the boto library and was tested with version
   http://code.google.com/p/boto/
 
 It also requires an account with Amazon Web Services (AWS) and the AWS S3 keys.
-The keys are added to your settings.py file, for example::
+Bucket name is required and cannot be empty.
+The keys and bucket name are added to your settings.py file, for example::
 
   # settings.py
   AWS_ACCESS_KEY_ID = ''
   AWS_SECRET_ACCESS_KEY = ''
+  AWS_BUCKET_NAME = 'bucket'
+
+Optional settings
+-----------------
+
+It is possible to customize sync_s3 directly from django settings file, for example::
+
+  # settings.py
+  AWS_S3_HOST = 'cs.example.com'
+  AWS_DEFAULT_ACL = 'private'
+  SYNC_MEDIA_S3_PREFIX = 'some_prefix'
+  FILTER_LIST = 'dir1, dir2'
+  AWS_CLOUDFRONT_DISTRIBUTION = 'E27LVI50CSW06W'
+  SYNC_S3_RENAME_GZIP_EXT = '.gz'