From 7a7d1ab0e6f976d014b8d7907d303e16825dd7ea Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Feb 2019 17:15:17 -0500 Subject: [PATCH 01/14] Scaffold Python package and move over source code * Create standard Python packaging artifacts * Move source code over from azavea/open-apparel-registry --- .gitignore | 118 ++++++++++++ LICENSE | 202 +++++++++++++++++++++ MANIFEST.in | 2 + README.md | 8 + ecsmanage/__init__.py | 0 ecsmanage/apps.py | 6 + ecsmanage/management/commands/__init__.py | 0 ecsmanage/management/commands/ecsmanage.py | 173 ++++++++++++++++++ setup.py | 46 +++++ 9 files changed, 555 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 ecsmanage/__init__.py create mode 100644 ecsmanage/apps.py create mode 100644 ecsmanage/management/commands/__init__.py create mode 100644 ecsmanage/management/commands/ecsmanage.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8b68fc0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,118 @@ + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +pyre/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..705e745 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Azavea Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..74215c3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..91b2a97 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# django-ecsmanage + +A Django app that provides a management command allowing you to run any +other management command on an AWS Elastic Container Service +(ECS) cluster. + +Useful for running migrations and other one-off commands in staging and +production environments. \ No newline at end of file diff --git a/ecsmanage/__init__.py b/ecsmanage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/apps.py b/ecsmanage/apps.py new file mode 100644 index 0000000..d95ac9e --- /dev/null +++ b/ecsmanage/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class EcsManageConfig(AppConfig): + name = 'ecsmanage' + verbose_name = 'ECS Manage' diff --git a/ecsmanage/management/commands/__init__.py b/ecsmanage/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py new file mode 100644 index 0000000..882d681 --- /dev/null +++ b/ecsmanage/management/commands/ecsmanage.py @@ -0,0 +1,173 @@ +import boto3 +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = 'Run a one-off management command on an ECS cluster.' + + def add_arguments(self, parser): + parser.add_argument( + '-e', '--env', + type=str, + choices=['staging', 'production'], + default='staging', + help="Environment to run the task in (staging or production)" + ) + + parser.add_argument( + 'cmd', + type=str, + nargs='+', + help="Command override for the ECS container (e.g. 'migrate')" + ) + + def handle(self, *args, **options): + """ + Run the given command on the latest app CLI task definition and print + out a URL to view the status. + """ + self.env = options['env'].title() + cmd = options['cmd'] + + self.ecs_client = boto3.client('ecs') + self.ec2_client = boto3.client('ec2') + + task_def_arn = self.get_task_def() + security_group_id = self.get_security_group() + subnet_id = self.get_subnet() + + task_id = self.run_task(task_def_arn, + security_group_id, + subnet_id, + cmd) + + url = ( + f'https://console.aws.amazon.com/ecs/home?region=us-east-1#' + f'/clusters/ecs{self.env}Cluster/tasks/{task_id}/details' + ) + + self.stdout.write(self.style.SUCCESS('Task started! View here:\n')) + self.stdout.write(self.style.SUCCESS(url)) + + def parse_response(self, response, key, idx=None): + """ + Perform a key-value lookup on a response from the AWS API, wrapping it + in error handling such that if the lookup fails the response body + will get propagated to the end user. + """ + if not response.get(key): + msg = f'Unexpected response from ECS API: {response}' + raise KeyError(msg) + else: + if idx is not None: + try: + return response[key][0] + except (IndexError, TypeError): + msg = ( + f"Unexpected value for '{key}' in response: " + f'{response}' + ) + raise IndexError(msg) + else: + return response[key] + + def get_task_def(self): + """ + Get the ARN of the latest ECS task definition for the app CLI. + """ + task_def_response = self.ecs_client.list_task_definitions( + familyPrefix=f'{self.env}AppCLI', + sort='DESC', + maxResults=1 + ) + + return self.parse_response(task_def_response, 'taskDefinitionArns', 0) + + def get_security_group(self): + """ + Get the ID of the security group to use for the app CLI. + """ + filters = [ + { + 'Name': 'tag:Name', + 'Values': ['sgAppEcsService'] + }, + { + 'Name': 'tag:Environment', + 'Values': [self.env] + }, + { + 'Name': 'tag:Project', + 'Values': ['OpenApparelRegistry'] + } + ] + + sg_response = self.ec2_client.describe_security_groups( + Filters=filters, + ) + + security_group = self.parse_response(sg_response, 'SecurityGroups', 0) + return security_group['GroupId'] + + def get_subnet(self): + """ + Get a subnet ID to use for the app CLI. + """ + filters = [ + { + 'Name': 'tag:Name', + 'Values': ['PrivateSubnet'] + }, + { + 'Name': 'tag:Environment', + 'Values': [self.env] + }, + { + 'Name': 'tag:Project', + 'Values': ['OpenApparelRegistry'] + } + ] + + subnet_response = self.ec2_client.describe_subnets( + Filters=filters, + ) + + subnet = self.parse_response(subnet_response, 'Subnets', 0) + return subnet['SubnetId'] + + def run_task(self, task_def_arn, security_group_id, subnet_id, cmd): + """ + Run a task for a given task definition ARN using the given security + group and subnets, and return the task ID. + """ + overrides = { + 'containerOverrides': [ + { + 'name': 'django', + 'command': cmd + } + ] + } + + network_configuration = { + 'awsvpcConfiguration': { + 'subnets': [subnet_id], + 'securityGroups': [security_group_id] + } + } + + task_response = self.ecs_client.run_task( + cluster=f'ecs{self.env}Cluster', + taskDefinition=task_def_arn, + overrides=overrides, + networkConfiguration=network_configuration, + count=1, + launchType='FARGATE' + ) + + task = self.parse_response(task_response, 'tasks', 0) + + # Parse the ask ARN, since ECS doesn't return the task ID. + # Task ARNS look like: arn:aws:ecs:::task/ + task_id = task['taskArn'].split('/')[1] + return task_id diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..08f3cbf --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +import os +from setuptools import find_packages, setup + +with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme: + README = readme.read() + +description = ( + 'Run any Django management command on an AWS Elastic Container Service (ECS)' + 'cluster.' +) + +# allow setup.py to be run from any path +os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + +setup( + name='django-ecsmanage', + version='0.0.0', + packages=find_packages(), + include_package_data=True, + license='Apache License 2.0', + description=description, + long_description=README, + long_description_content_type='text/markdown', + url='https://github.com/azavea/django-ecsmanage/', + author='Azavea', + author_email='yourname@example.com', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + install_requires=[ + 'Django >=1.11, <=2.1', + 'boto3 >=1.9.0' + ] + classifiers=[ + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache License 2.0', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + ], +) \ No newline at end of file From 3dd71165a926b89264842ea64d6a741606013d58 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Feb 2019 18:07:09 -0500 Subject: [PATCH 02/14] Add PR template and test artifacts * Sketch out test infrastructure with testing artifacts * Add a PR template to the repo --- .github/PULL_REQUEST_TEMPLATE.md | 21 ++++++++++++++ ecsmanage/tests.py | 0 scripts/test | 49 ++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 ecsmanage/tests.py create mode 100755 scripts/test diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..997e6b6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Overview + +_Add a brief description of what this PR does, and why it is needed._ + +Closes #XXX + +### Demo + +_Optional. Screenshots, `curl` examples, etc._ + +### Notes + +_Optional. Ancillary topics, caveats, alternative strategies that didn't work out, anything else._ + +## Testing Instructions + + * How to test this PR + * Prefer bulleted description + * Start after checking out this branch + * Include any setup required, such as bundling scripts, restarting services, etc. + * Include test case, and expected output \ No newline at end of file diff --git a/ecsmanage/tests.py b/ecsmanage/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000..c0b8046 --- /dev/null +++ b/scripts/test @@ -0,0 +1,49 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECSMANAGE_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") [OPTIONS] + +Run tests for the django-ecsmanage app. + +Options: + --help Display help text. + --lint Run shell and Python linters. + --app Run app tests. +" + +function run_linters() { + # Lint Bash scripts. + if command -v shellcheck > /dev/null; then + shellcheck scripts/* + fi + + # Lint Python scripts. + # Lint Python scripts. + docker-compose run --rm --no-deps \ + --entrypoint flake8 app \ + --exclude *.pyc +} + +function run_tests() { + docker-compose run --rm --entrypoint python app manage.py test --noinput +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + elif [ "${1:-}" = "--lint" ]; then + run_linters + elif [ "${1:-}" = "--app" ]; then + run_tests + else + run_linters + run_tests + fi +fi \ No newline at end of file From a35b00ecbe2c14b3cdc717f660879c980685e9ef Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Feb 2019 18:07:35 -0500 Subject: [PATCH 03/14] Adjust ecsmanage command to read configuration values Read configurations for the management command from a settings attribute instead of hardcoding AWS resource names. --- ecsmanage/management/commands/ecsmanage.py | 103 ++++++++++++++------- 1 file changed, 67 insertions(+), 36 deletions(-) diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py index 882d681..f8989a4 100644 --- a/ecsmanage/management/commands/ecsmanage.py +++ b/ecsmanage/management/commands/ecsmanage.py @@ -1,5 +1,6 @@ import boto3 -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings class Command(BaseCommand): @@ -9,9 +10,8 @@ def add_arguments(self, parser): parser.add_argument( '-e', '--env', type=str, - choices=['staging', 'production'], - default='staging', - help="Environment to run the task in (staging or production)" + default='default', + help="Environment to run the task in, as defined in ECS_ENVIRONMENTS." ) parser.add_argument( @@ -26,29 +26,76 @@ def handle(self, *args, **options): Run the given command on the latest app CLI task definition and print out a URL to view the status. """ - self.env = options['env'].title() + self.env = options['env'] cmd = options['cmd'] + config = self.parse_config() + self.ecs_client = boto3.client('ecs') self.ec2_client = boto3.client('ec2') - task_def_arn = self.get_task_def() - security_group_id = self.get_security_group() - subnet_id = self.get_subnet() + task_def_arn = self.get_task_def(config['TASK_DEFINITION_NAME']) + security_group_id = self.get_security_group(config['SECURITY_GROUP_NAME']) + subnet_id = self.get_subnet(config['SUBNET_NAME']) - task_id = self.run_task(task_def_arn, + task_id = self.run_task(config, + task_def_arn, security_group_id, subnet_id, cmd) + cluster_name = config['CLUSTER_NAME'] + aws_region = config['AWS_REGION'] + url = ( - f'https://console.aws.amazon.com/ecs/home?region=us-east-1#' - f'/clusters/ecs{self.env}Cluster/tasks/{task_id}/details' + f'https://console.aws.amazon.com/ecs/home?region={aws_region}#' + f'/clusters/{cluster_name}/tasks/{task_id}/details' ) self.stdout.write(self.style.SUCCESS('Task started! View here:\n')) self.stdout.write(self.style.SUCCESS(url)) + def parse_config(self): + """ + Parse configuration settings for the app, checking to make sure that + they're valid. + """ + if getattr(settings, 'ECS_ENVIRONMENTS') is None: + raise CommandError( + 'ECS_ENVIRONMENTS was not found in the Django settings.' + ) + + ecs_configs = settings.ECS_ENVIRONMENTS.get(self.env, None) + if ecs_configs is None: + raise CommandError( + f'Environment "{self.env}" is not a recognized environment in ' + 'ECS_ENVIRONMENTS (environments include: ' + f'{ECS_ENVIRONMENTS.keys()})' + ) + + config = { + 'TASK_DEFINITION_NAME': '', + 'CLUSTER_NAME': '', + 'LAUNCH_TYPE': 'FARGATE', + 'SECURITY_GROUP_NAME': '', + 'SUBNET_NAME': '', + 'AWS_REGION': 'us-east-1', + } + + for config_name, config_default in config.keys(): + if ecs_configs.get(config_name) is None: + if config_default == '': + raise CommandError( + f'Environment "{self.env}" is missing required config ' + f'attribute {config_name}' + ) + else: + config[config_name] = config_default + else: + config[config_name] = ecs_configs[config_name] + + return config + def parse_response(self, response, key, idx=None): """ Perform a key-value lookup on a response from the AWS API, wrapping it @@ -71,35 +118,27 @@ def parse_response(self, response, key, idx=None): else: return response[key] - def get_task_def(self): + def get_task_def(self, task_def_name): """ Get the ARN of the latest ECS task definition for the app CLI. """ task_def_response = self.ecs_client.list_task_definitions( - familyPrefix=f'{self.env}AppCLI', + familyPrefix=task_def_name, sort='DESC', maxResults=1 ) return self.parse_response(task_def_response, 'taskDefinitionArns', 0) - def get_security_group(self): + def get_security_group(self, security_group_name): """ Get the ID of the security group to use for the app CLI. """ filters = [ { 'Name': 'tag:Name', - 'Values': ['sgAppEcsService'] - }, - { - 'Name': 'tag:Environment', - 'Values': [self.env] + 'Values': [security_group_name] }, - { - 'Name': 'tag:Project', - 'Values': ['OpenApparelRegistry'] - } ] sg_response = self.ec2_client.describe_security_groups( @@ -109,23 +148,15 @@ def get_security_group(self): security_group = self.parse_response(sg_response, 'SecurityGroups', 0) return security_group['GroupId'] - def get_subnet(self): + def get_subnet(self, subnet_name): """ Get a subnet ID to use for the app CLI. """ filters = [ { 'Name': 'tag:Name', - 'Values': ['PrivateSubnet'] + 'Values': [subnet_name] }, - { - 'Name': 'tag:Environment', - 'Values': [self.env] - }, - { - 'Name': 'tag:Project', - 'Values': ['OpenApparelRegistry'] - } ] subnet_response = self.ec2_client.describe_subnets( @@ -135,7 +166,7 @@ def get_subnet(self): subnet = self.parse_response(subnet_response, 'Subnets', 0) return subnet['SubnetId'] - def run_task(self, task_def_arn, security_group_id, subnet_id, cmd): + def run_task(self, config, task_def_arn, security_group_id, subnet_id, cmd): """ Run a task for a given task definition ARN using the given security group and subnets, and return the task ID. @@ -157,12 +188,12 @@ def run_task(self, task_def_arn, security_group_id, subnet_id, cmd): } task_response = self.ecs_client.run_task( - cluster=f'ecs{self.env}Cluster', + cluster=config['CLUSTER_NAME'], taskDefinition=task_def_arn, overrides=overrides, networkConfiguration=network_configuration, count=1, - launchType='FARGATE' + launchType=config['LAUNCH_TYPE'] ) task = self.parse_response(task_response, 'tasks', 0) From dbe4a70d4d14977e53e93414b045ad90c39fa55f Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Feb 2019 22:23:56 -0500 Subject: [PATCH 04/14] Get ecsmanage command working Polish up ecsmanage.py management command script to get the module working with OAR. --- ecsmanage/management/commands/ecsmanage.py | 64 +++++++++++----------- setup.py | 2 +- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py index f8989a4..b2231e3 100644 --- a/ecsmanage/management/commands/ecsmanage.py +++ b/ecsmanage/management/commands/ecsmanage.py @@ -11,7 +11,7 @@ def add_arguments(self, parser): '-e', '--env', type=str, default='default', - help="Environment to run the task in, as defined in ECS_ENVIRONMENTS." + help="Environment to run the task in, as defined in ECSMANAGE_ENVIRONMENTS." ) parser.add_argument( @@ -35,8 +35,10 @@ def handle(self, *args, **options): self.ec2_client = boto3.client('ec2') task_def_arn = self.get_task_def(config['TASK_DEFINITION_NAME']) - security_group_id = self.get_security_group(config['SECURITY_GROUP_NAME']) - subnet_id = self.get_subnet(config['SUBNET_NAME']) + security_group_id = self.get_security_group( + config['SECURITY_GROUP_TAGS'] + ) + subnet_id = self.get_subnet(config['SUBNET_TAGS']) task_id = self.run_task(config, task_def_arn, @@ -60,29 +62,29 @@ def parse_config(self): Parse configuration settings for the app, checking to make sure that they're valid. """ - if getattr(settings, 'ECS_ENVIRONMENTS') is None: + if getattr(settings, 'ECSMANAGE_ENVIRONMENTS') is None: raise CommandError( - 'ECS_ENVIRONMENTS was not found in the Django settings.' + 'ECSMANAGE_ENVIRONMENTS was not found in the Django settings.' ) - ecs_configs = settings.ECS_ENVIRONMENTS.get(self.env, None) + ecs_configs = settings.ECSMANAGE_ENVIRONMENTS.get(self.env, None) if ecs_configs is None: raise CommandError( f'Environment "{self.env}" is not a recognized environment in ' - 'ECS_ENVIRONMENTS (environments include: ' - f'{ECS_ENVIRONMENTS.keys()})' + 'ECSMANAGE_ENVIRONMENTS (environments include: ' + f'{ECSMANAGE_ENVIRONMENTS.keys()})' ) config = { 'TASK_DEFINITION_NAME': '', 'CLUSTER_NAME': '', + 'SECURITY_GROUP_TAGS': '', + 'SUBNET_TAGS': '', 'LAUNCH_TYPE': 'FARGATE', - 'SECURITY_GROUP_NAME': '', - 'SUBNET_NAME': '', 'AWS_REGION': 'us-east-1', } - for config_name, config_default in config.keys(): + for config_name, config_default in config.items(): if ecs_configs.get(config_name) is None: if config_default == '': raise CommandError( @@ -130,38 +132,34 @@ def get_task_def(self, task_def_name): return self.parse_response(task_def_response, 'taskDefinitionArns', 0) - def get_security_group(self, security_group_name): + def get_security_group(self, security_group_tags): """ Get the ID of the security group to use for the app CLI. """ - filters = [ - { - 'Name': 'tag:Name', - 'Values': [security_group_name] - }, - ] - - sg_response = self.ec2_client.describe_security_groups( - Filters=filters, - ) + filters = [] + for tagname, tagvalue in security_group_tags.items(): + filters.append({ + 'Name': f'tag:{tagname}', + 'Values': [tagvalue] + }) + + sg_response = self.ec2_client.describe_security_groups(Filters=filters) security_group = self.parse_response(sg_response, 'SecurityGroups', 0) return security_group['GroupId'] - def get_subnet(self, subnet_name): + def get_subnet(self, subnet_tags): """ Get a subnet ID to use for the app CLI. """ - filters = [ - { - 'Name': 'tag:Name', - 'Values': [subnet_name] - }, - ] - - subnet_response = self.ec2_client.describe_subnets( - Filters=filters, - ) + filters = [] + for tagname, tagvalue in subnet_tags.items(): + filters.append({ + 'Name': f'tag:{tagname}', + 'Values': [tagvalue] + }) + + subnet_response = self.ec2_client.describe_subnets(Filters=filters) subnet = self.parse_response(subnet_response, 'Subnets', 0) return subnet['SubnetId'] diff --git a/setup.py b/setup.py index 08f3cbf..314ff01 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ install_requires=[ 'Django >=1.11, <=2.1', 'boto3 >=1.9.0' - ] + ], classifiers=[ 'Environment :: Web Environment', 'Framework :: Django', From 159e6547b1daf0df36bbb01217912d3887d542d5 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Feb 2019 22:49:10 -0500 Subject: [PATCH 05/14] Make management command installable --- ecsmanage/management/__init__.py | 0 ecsmanage/management/commands/ecsmanage.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 ecsmanage/management/__init__.py diff --git a/ecsmanage/management/__init__.py b/ecsmanage/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py index b2231e3..74adea8 100644 --- a/ecsmanage/management/commands/ecsmanage.py +++ b/ecsmanage/management/commands/ecsmanage.py @@ -72,7 +72,7 @@ def parse_config(self): raise CommandError( f'Environment "{self.env}" is not a recognized environment in ' 'ECSMANAGE_ENVIRONMENTS (environments include: ' - f'{ECSMANAGE_ENVIRONMENTS.keys()})' + f'{settings.ECSMANAGE_ENVIRONMENTS.keys()})' ) config = { From a46dc8d93acfab5050e99dc7483f9f48944cc333 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Thu, 28 Feb 2019 11:56:52 -0500 Subject: [PATCH 06/14] Initialize tests Set up some simple tests for the package configuration. --- .gitignore | 7 +- ecsmanage/management/commands/ecsmanage.py | 14 +++- scripts/test | 9 +-- scripts/update | 35 ++++++++ setup.py | 7 +- ecsmanage/tests.py => tests/__init__.py | 0 tests/settings_test.py | 4 + tests/test_configuration.py | 93 ++++++++++++++++++++++ 8 files changed, 155 insertions(+), 14 deletions(-) create mode 100755 scripts/update rename ecsmanage/tests.py => tests/__init__.py (100%) create mode 100644 tests/settings_test.py create mode 100644 tests/test_configuration.py diff --git a/.gitignore b/.gitignore index 8b68fc0..0312f52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ - - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -115,4 +113,7 @@ venv.bak/ dmypy.json # Pyre type checker -pyre/ \ No newline at end of file +pyre/ + +# Editors +.vscode diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py index 74adea8..5fa4d69 100644 --- a/ecsmanage/management/commands/ecsmanage.py +++ b/ecsmanage/management/commands/ecsmanage.py @@ -11,7 +11,10 @@ def add_arguments(self, parser): '-e', '--env', type=str, default='default', - help="Environment to run the task in, as defined in ECSMANAGE_ENVIRONMENTS." + help=( + 'Environment to run the task in, as defined in' + 'ECSMANAGE_ENVIRONMENTS.' + ) ) parser.add_argument( @@ -74,7 +77,7 @@ def parse_config(self): 'ECSMANAGE_ENVIRONMENTS (environments include: ' f'{settings.ECSMANAGE_ENVIRONMENTS.keys()})' ) - + config = { 'TASK_DEFINITION_NAME': '', 'CLUSTER_NAME': '', @@ -164,7 +167,12 @@ def get_subnet(self, subnet_tags): subnet = self.parse_response(subnet_response, 'Subnets', 0) return subnet['SubnetId'] - def run_task(self, config, task_def_arn, security_group_id, subnet_id, cmd): + def run_task(self, + config, + task_def_arn, + security_group_id, + subnet_id, + cmd): """ Run a task for a given task definition ARN using the given security group and subnets, and return the task ID. diff --git a/scripts/test b/scripts/test index c0b8046..95787d1 100755 --- a/scripts/test +++ b/scripts/test @@ -17,6 +17,7 @@ Options: --lint Run shell and Python linters. --app Run app tests. " +} function run_linters() { # Lint Bash scripts. @@ -25,14 +26,12 @@ function run_linters() { fi # Lint Python scripts. - # Lint Python scripts. - docker-compose run --rm --no-deps \ - --entrypoint flake8 app \ - --exclude *.pyc + ./.venv/bin/flake8 --exclude ./*.pyc,./.venv } function run_tests() { - docker-compose run --rm --entrypoint python app manage.py test --noinput + PYTHONPATH="./tests/" DJANGO_SETTINGS_MODULE='settings_test' \ + ./.venv/bin/django-admin test --noinput } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then diff --git a/scripts/update b/scripts/update new file mode 100755 index 0000000..e290047 --- /dev/null +++ b/scripts/update @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECSMANAGE_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") + +Install the package for testing. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + if ! [ -x "$(command -v python3)" ]; then + echo "Error: python3 is not installed." + exit 1j + elif ! [ -x "$(command -v pip3)" ]; then + echo "Error: pip3 is not installed." + exit 1 + else + if ! [ -d ".venv" ]; then + python3 -m venv .venv + else + ./.venv/bin/pip3 install -e ".[tests]" + fi + fi + fi +fi \ No newline at end of file diff --git a/setup.py b/setup.py index 314ff01..b570fea 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ README = readme.read() description = ( - 'Run any Django management command on an AWS Elastic Container Service (ECS)' - 'cluster.' + 'Run any Django management command on an AWS Elastic Container Service' + '(ECS) cluster.' ) # allow setup.py to be run from any path @@ -29,6 +29,7 @@ 'Django >=1.11, <=2.1', 'boto3 >=1.9.0' ], + extras_require={'tests': ['moto >= 1.3.3', 'flake8 >= 3.7.7']}, classifiers=[ 'Environment :: Web Environment', 'Framework :: Django', @@ -43,4 +44,4 @@ 'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', ], -) \ No newline at end of file +) diff --git a/ecsmanage/tests.py b/tests/__init__.py similarity index 100% rename from ecsmanage/tests.py rename to tests/__init__.py diff --git a/tests/settings_test.py b/tests/settings_test.py new file mode 100644 index 0000000..1b5928f --- /dev/null +++ b/tests/settings_test.py @@ -0,0 +1,4 @@ +""" +Django settings for testing purposes. +""" +SECRET_KEY = 'testing!' diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..6280722 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,93 @@ +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import SimpleTestCase + + +class ConfigurationTestCase(SimpleTestCase): + """ + Test the configuration of settings for the management command. + """ + def test_failure_when_no_settings(self): + """ + Test that the command throws an error when the ECSMANAGE_ENVIRONMENTS + setting does not exist. + """ + with self.assertRaises(CommandError): + call_command('ecsmanage', 'help') + + def test_failure_when_missing_environment(self): + """ + Test that the command throws an error when the environment passed to + the CLI does not exist in ECSMANAGE_ENVIRONMENTS. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': {}, + 'production': {} + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='foobar') + + def test_failure_when_no_task_def_name(self): + """ + Test that the command throws an error when the configuration is missing + a task definition name. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'CLUSTER_NAME': 'foo', + 'SECURITY_GROUP_TAGS': {}, + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_cluster_name(self): + """ + Test that the command throws an error when the configuration is missing + a cluster name. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'SECURITY_GROUP_TAGS': {}, + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_security_group_tags(self): + """ + Test that the command throws an error when the configuration is missing + security group tags. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'CLUSTER_NAME': 'bar', + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_subnet_tags(self): + """ + Test that the command throws an error when the configuration is missing + subnet tags. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'CLUSTER_NAME': 'bar', + 'SECURITY_GROUP_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') From b2e542c7d4ce92ac79ebba080d5c037d4994bcdb Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Thu, 28 Feb 2019 11:57:13 -0500 Subject: [PATCH 07/14] Draft simple README Draft basic instructions for setup and development in a README. --- README.md | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 91b2a97..63d6ebf 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,137 @@ other management command on an AWS Elastic Container Service (ECS) cluster. Useful for running migrations and other one-off commands in staging and -production environments. \ No newline at end of file +production environments. With `django-ecsmanage`, you can easily run migrations +on a remote cluster from the command line: + +``` +$ django-admin ecsmanage migrate +``` + +## Contents + +- [Installation](#installation) +- [Configuration](#configuration) + - [Environments](#environments) + - [AWS Resources](#aws-resources) +- [Developing](#developing) + +## Installation + +Install from GitHub using pip: + +``` +$ pip install git+https://github.com/azavea/django-ecsmanage.git +``` + +Update `INSTALLED_APPS` in your Django settings to install the app: + +```python +INSTALLED_APPS = ( + ... + 'ecsmanage', +) +``` + +## Configuration + +Settings for the management command are kept in a single configuration +dictionary in your Django settings named `ECSMANAGE_ENVIRONMENTS`. Each entry in +`ECSMANAGE_ENVIRONMENTS` should be a key-value pair corresponding to a +named environment (like `default` or `production`) and a set of AWS resources +in that environment. For example: + +```python +ECSMANAGE_ENVIRONMENTS = { + 'default': { + 'TASK_DEFINITION_NAME': 'StagingAppCLI', + 'CLUSTER_NAME': 'ecsStagingCluster', + 'LAUNCH_TYPE': 'FARGATE', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgAppEcsService', + 'Environment': 'Staging', + 'Project': 'ProjectName' + }, + 'SUBNET_TAGS': { + 'Name': 'PrivateSubnet', + 'Environment': 'Staging', + 'Project': 'ProjectName' + }, + 'AWS_REGION': 'us-east-1', + }, +} +``` + +### Environments + +The key name for an environment can be any string. You can use this name +with the `--env` flag when running the command to run a command on a +different environment. Take this `ECSMANAGE_ENVIRONMENTS` variable +as an example: + +```python +ECSMANAGE_ENVIRONMENTS = { + 'default': { + 'TASK_DEFINITION_NAME': 'StagingAppCLI', + 'CLUSTER_NAME': 'ecsStagingCluster', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgStagingAppEcsService', + }, + 'SUBNET_TAGS': { + 'Name': 'StagingPrivateSubnet', + }, + }, + 'production': { + 'TASK_DEFINITION_NAME': 'ProductionAppCLI', + 'CLUSTER_NAME': 'ecsProductionCluster', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgProductionAppEcsService', + }, + 'SUBNET_TAGS': { + 'Name': 'ProductionPrivateSubnet', + }, + }, +} +``` + +Using the above settings, you could run production migrations with the +following command: + +``` +$ django-admin ecsmanage --env production migrate +``` + +If the `--env` argument is not present, the command will default to the +environment named `default`. + +### AWS Resources + +The following keys in an environment help the management command locate +the appropriate AWS resources for your cluster: + +| key name | description | default | +| -------- | ----------- | ------- | +| `TASK_DEFINITION_NAME` | The name of your ECS task definition. The command will automatically retrieve the latest definition. | | +| `CLUSTER_NAME` | The name of your ECS cluster. | | +| `SECURITY_GROUP_TAGS` | A dictionary of tags to use to identify a security group for your task. | | +| `SUBNET_TAGS` | A dictionary of tags to use to identify a subnet for your task. | | +| `LAUNCH_TYPE` | The ECS launch type for your task. | `FARGATE` | +| `AWS_REGION` | The AWS region to run your task. | `us-east-1` | + +## Developing + +Local development is managed with Python virtual environments. Make sure that +you have [Python 3.4+ and pip installed](https://www.python.org/downloads/) +before starting. + +Install the development package in a virtual environment: + +``` +$ ./scripts/update +``` + +Run the tests: + +``` +$ ./scripts/test +``` \ No newline at end of file From 4096b3fd6269cd0d355998e25fc2469b5e812045 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Thu, 28 Feb 2019 13:22:24 -0500 Subject: [PATCH 08/14] Add Travis config and drop support for py2.7 + py3.5 * Add .travis.yml for CI * Drop support for py2.7 (EOL in Jan) and py3.5 (f strings) --- .travis.yml | 26 ++++++++++++++++++++++++++ setup.py | 4 +--- 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..445d826 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +cache: pip + +python: + - "3.6" + - "3.7" + +before_install: + # Work around Travis environment. Taken from: + # https://github.com/azavea/django-amazon-ses/blob/57ed8619435194742308c6f15cbf9c91b231b88a/.travis.yml#L19 + - sudo rm /etc/boto.cfg + +install: ./scripts/update + +script: ./scripts/test + +deploy: + provider: pypi + user: azavea + password: + secure: fG1z1XACymeQVTJRsUUqqQ7NXYrN//82LIRuCJknbwsFIMGBPyuiQyOkGul/BJ8N4lPWFltTOwRKbF9YhJVY3OOfs6/BXc6A4W49pejr53m1ECc8JJNJ3AL9cChBK2rjjr/NOzvAeZdBGjZSmd37Z486PsJiA1FJ+/sTQq24SZ2CxwUYnEtgnzKDB7F2B3Iyy98SfIOnIiLGAHj/vNjl0wpj1xFfqZmIcujj7HBuAHxyEajDfCk8z38ukZht5tUbl1sY4dU5YfFvsmIG73DaEooJ6wGxQUaw5b7+1NZRL/ic2IXnVoNi0uFrvFKkUYnII3aniSIr7UDwLUfLAb8vgHW+UYcv/gCjNH76+Jr5U2hlLJ9ZDj4eB+mcTak3B6ttHlWXPlDsTX3cMwxaqTp/cKV4VKaSVloFcc9714e2iMrLTpTP6qcQ4OmkDxH223qyoWg2TrE1CF3muJ2sxnsVqiSIWB45v3n1dtvhPZ9zW+FDggsk1N0CUdPV351AdxZHJyJl9lEdgRqBlr/dhgGf412RGDieNY3USw2UPR4ZnVWNBbsQq7ScU0MrVb7H/Y73sULH1Yh4WY8kt5cM1FoxVMS/i41/jh7Ttm0HCY5p/POk5v7R8ROLe6J0gCFyyiIsvV6XODf+fAgkCOTIC1/hzHEH5HdsGGkBV5vyTYzNhO0= + distributions: sdist bdist_wheel + on: + tags: true + python: "3.7" + repo: azavea/django-ecsmanage diff --git a/setup.py b/setup.py index b570fea..57b6f8e 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url='https://github.com/azavea/django-ecsmanage/', author='Azavea', author_email='yourname@example.com', - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=3.5', install_requires=[ 'Django >=1.11, <=2.1', 'boto3 >=1.9.0' @@ -37,8 +37,6 @@ 'License :: OSI Approved :: Apache License 2.0', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', From f42609d3676cc55a8429a5084daa4bd1575c0b2f Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Thu, 28 Feb 2019 14:56:23 -0500 Subject: [PATCH 09/14] Clean up Travis config Squash a bunch of commits designed to appease Travis. --- .travis.yml | 54 ++++++++++++++++++++++++++++++-------------------- scripts/test | 8 ++++---- scripts/update | 6 +++--- setup.py | 1 - 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 445d826..6af989a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,36 @@ language: python cache: pip - -python: - - "3.6" - - "3.7" - +env: + global: + - PYTHONPATH="./tests/" + - DJANGO_SETTINGS_MODULE="settings_test" +matrix: + include: + - python: 3.6 + - python: 3.7 + dist: xenial + sudo: true before_install: - # Work around Travis environment. Taken from: - # https://github.com/azavea/django-amazon-ses/blob/57ed8619435194742308c6f15cbf9c91b231b88a/.travis.yml#L19 - - sudo rm /etc/boto.cfg - -install: ./scripts/update - -script: ./scripts/test - +- sudo rm /etc/boto.cfg +install: pip install .[tests] +script: django-admin test --noinput deploy: - provider: pypi - user: azavea - password: - secure: fG1z1XACymeQVTJRsUUqqQ7NXYrN//82LIRuCJknbwsFIMGBPyuiQyOkGul/BJ8N4lPWFltTOwRKbF9YhJVY3OOfs6/BXc6A4W49pejr53m1ECc8JJNJ3AL9cChBK2rjjr/NOzvAeZdBGjZSmd37Z486PsJiA1FJ+/sTQq24SZ2CxwUYnEtgnzKDB7F2B3Iyy98SfIOnIiLGAHj/vNjl0wpj1xFfqZmIcujj7HBuAHxyEajDfCk8z38ukZht5tUbl1sY4dU5YfFvsmIG73DaEooJ6wGxQUaw5b7+1NZRL/ic2IXnVoNi0uFrvFKkUYnII3aniSIr7UDwLUfLAb8vgHW+UYcv/gCjNH76+Jr5U2hlLJ9ZDj4eB+mcTak3B6ttHlWXPlDsTX3cMwxaqTp/cKV4VKaSVloFcc9714e2iMrLTpTP6qcQ4OmkDxH223qyoWg2TrE1CF3muJ2sxnsVqiSIWB45v3n1dtvhPZ9zW+FDggsk1N0CUdPV351AdxZHJyJl9lEdgRqBlr/dhgGf412RGDieNY3USw2UPR4ZnVWNBbsQq7ScU0MrVb7H/Y73sULH1Yh4WY8kt5cM1FoxVMS/i41/jh7Ttm0HCY5p/POk5v7R8ROLe6J0gCFyyiIsvV6XODf+fAgkCOTIC1/hzHEH5HdsGGkBV5vyTYzNhO0= - distributions: sdist bdist_wheel - on: - tags: true - python: "3.7" - repo: azavea/django-ecsmanage + - provider: pypi + user: django-ecsmanage + skip_existing: true + server: https://test.pypi.org/legacy/ + password: + secure: 0x32eNc+iKoYxM+nvMEEQJIoPHq3ehNqVUyKMT6crntYBBC8xgD2NHwsF0HCtfgm/NitWFlZLwtPhpb8dxTW//hh5gafkLt4/MALHFYBeRYm0AihCfTkjiuC/FCkpQ9K7X9dVO5qQN5gi5PQvkFW13NrKWOUCMTr2/jE8mLaRS3exCKUUamsd/S6BxgXY7LHm50ueDFDTYSkZrqKwIHnwwpMLcLvZZxcRJXZxpK567e8KTpcBelG+/ecY+Cn/+L0JJ29p6CGWCuW+vaIeYaeHAU6XZZ/cqNuycYlFTX+CpKSlajQgFQTMC1O1L6OVX73gSpZ2jUSd5dSimDmxgttsoo+kCxPA/hY9cUfXUpS1c52xVv9ZXZgweIV0g/EXcKLe1SIal8wy6fTE5DFzyZsZutE6ipksZjgrsFM26IHPflsf/caM/n3nDqoeXW5FtA8t4wpsemYWRFnYhCbWUWpiqAweKzzK/+3AJKEf55d4Zk7TokEjg5bZdBLroI0zy4MM6clxzujZNpn8FgTtlKhN9sDtHU5AvSe0+BUJRHJQVVSbc0wk5L+03dYarPlhShIWISSjoLFgDJhJPCZLgWD1OTetv0IekxU38AJnyqUi5XvUUbtOOiyNyU6xCFVp2XMQdRwMv41hYvEFbY4Su1B3AF2vSJCvK1HQehvzAMt4JE= + on: + distributions: sdist bdist_wheel + repo: azavea/django-ecsmanage + branch: develop + - provider: pypi + user: django-ecsmanage + skip_existing: true + password: + secure: 0x32eNc+iKoYxM+nvMEEQJIoPHq3ehNqVUyKMT6crntYBBC8xgD2NHwsF0HCtfgm/NitWFlZLwtPhpb8dxTW//hh5gafkLt4/MALHFYBeRYm0AihCfTkjiuC/FCkpQ9K7X9dVO5qQN5gi5PQvkFW13NrKWOUCMTr2/jE8mLaRS3exCKUUamsd/S6BxgXY7LHm50ueDFDTYSkZrqKwIHnwwpMLcLvZZxcRJXZxpK567e8KTpcBelG+/ecY+Cn/+L0JJ29p6CGWCuW+vaIeYaeHAU6XZZ/cqNuycYlFTX+CpKSlajQgFQTMC1O1L6OVX73gSpZ2jUSd5dSimDmxgttsoo+kCxPA/hY9cUfXUpS1c52xVv9ZXZgweIV0g/EXcKLe1SIal8wy6fTE5DFzyZsZutE6ipksZjgrsFM26IHPflsf/caM/n3nDqoeXW5FtA8t4wpsemYWRFnYhCbWUWpiqAweKzzK/+3AJKEf55d4Zk7TokEjg5bZdBLroI0zy4MM6clxzujZNpn8FgTtlKhN9sDtHU5AvSe0+BUJRHJQVVSbc0wk5L+03dYarPlhShIWISSjoLFgDJhJPCZLgWD1OTetv0IekxU38AJnyqUi5XvUUbtOOiyNyU6xCFVp2XMQdRwMv41hYvEFbY4Su1B3AF2vSJCvK1HQehvzAMt4JE= + on: + distributions: sdist bdist_wheel + repo: azavea/django-ecsmanage + branch: master diff --git a/scripts/test b/scripts/test index 95787d1..9cd87c6 100755 --- a/scripts/test +++ b/scripts/test @@ -9,7 +9,7 @@ fi function usage() { echo -n \ "Usage: $(basename "$0") [OPTIONS] - + Run tests for the django-ecsmanage app. Options: @@ -23,14 +23,14 @@ function run_linters() { # Lint Bash scripts. if command -v shellcheck > /dev/null; then shellcheck scripts/* - fi + fi # Lint Python scripts. ./.venv/bin/flake8 --exclude ./*.pyc,./.venv } function run_tests() { - PYTHONPATH="./tests/" DJANGO_SETTINGS_MODULE='settings_test' \ + PYTHONPATH="./tests/" DJANGO_SETTINGS_MODULE="settings_test" \ ./.venv/bin/django-admin test --noinput } @@ -45,4 +45,4 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then run_linters run_tests fi -fi \ No newline at end of file +fi diff --git a/scripts/update b/scripts/update index e290047..b778c9c 100755 --- a/scripts/update +++ b/scripts/update @@ -9,7 +9,7 @@ fi function usage() { echo -n \ "Usage: $(basename "$0") - + Install the package for testing. " } @@ -20,7 +20,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then else if ! [ -x "$(command -v python3)" ]; then echo "Error: python3 is not installed." - exit 1j + exit 1 elif ! [ -x "$(command -v pip3)" ]; then echo "Error: pip3 is not installed." exit 1 @@ -32,4 +32,4 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then fi fi fi -fi \ No newline at end of file +fi diff --git a/setup.py b/setup.py index 57b6f8e..832da94 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ 'Environment :: Web Environment', 'Framework :: Django', 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache License 2.0', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3.6', From f9290231fc5e347f359f0ae19e178088be0122e0 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Sat, 2 Mar 2019 14:57:42 -0500 Subject: [PATCH 10/14] Document workarounds in .travis.yml Make sure workarounds for py3.7 and Boto3 testing are documented. --- .travis.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6af989a..2929cc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,28 @@ language: python cache: pip + env: global: - - PYTHONPATH="./tests/" - - DJANGO_SETTINGS_MODULE="settings_test" + - PYTHONPATH="./tests/" + - DJANGO_SETTINGS_MODULE="settings_test" + matrix: include: - - python: 3.6 - - python: 3.7 - dist: xenial - sudo: true -before_install: -- sudo rm /etc/boto.cfg + - python: 3.6 + # Workaround for Python 3.7 support. See: + # https://github.com/travis-ci/travis-ci/issues/9815 + - python: 3.7 + dist: xenial + sudo: true + +# Workaround for testing Boto3 on Travis. Taken from: +# https://github.com/azavea/django-amazon-ses/blob/57ed8619435194742308c6f15cbf9c91b231b88a/.travis.yml#L17-L19 +before_install: sudo rm /etc/boto.cfg + install: pip install .[tests] + script: django-admin test --noinput + deploy: - provider: pypi user: django-ecsmanage From 80c3ba5af142438be9eefa1d527ad4fe78fe3f20 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Sat, 2 Mar 2019 15:13:40 -0500 Subject: [PATCH 11/14] Clean up README Add additional notes and context to the README. --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 63d6ebf..a96685e 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ A Django app that provides a management command allowing you to run any other management command on an AWS Elastic Container Service (ECS) cluster. -Useful for running migrations and other one-off commands in staging and -production environments. With `django-ecsmanage`, you can easily run migrations +With `django-ecsmanage`, you can easily run migrations and other one-off tasks on a remote cluster from the command line: ``` @@ -22,10 +21,10 @@ $ django-admin ecsmanage migrate ## Installation -Install from GitHub using pip: +Install from PyPi using pip: ``` -$ pip install git+https://github.com/azavea/django-ecsmanage.git +$ pip install django-ecsmanage ``` Update `INSTALLED_APPS` in your Django settings to install the app: @@ -43,7 +42,7 @@ Settings for the management command are kept in a single configuration dictionary in your Django settings named `ECSMANAGE_ENVIRONMENTS`. Each entry in `ECSMANAGE_ENVIRONMENTS` should be a key-value pair corresponding to a named environment (like `default` or `production`) and a set of AWS resources -in that environment. For example: +associated with that environment. For example: ```python ECSMANAGE_ENVIRONMENTS = { @@ -66,11 +65,14 @@ ECSMANAGE_ENVIRONMENTS = { } ``` +This configuration defines a single environment, named `default`, with +associated AWS ECS resources. + ### Environments The key name for an environment can be any string. You can use this name with the `--env` flag when running the command to run a command on a -different environment. Take this `ECSMANAGE_ENVIRONMENTS` variable +different environment. Take this `ECSMANAGE_ENVIRONMENTS` configuration as an example: ```python @@ -98,6 +100,7 @@ ECSMANAGE_ENVIRONMENTS = { } ``` +This configuration defines two environments, `default` and `production`. Using the above settings, you could run production migrations with the following command: @@ -110,7 +113,7 @@ environment named `default`. ### AWS Resources -The following keys in an environment help the management command locate +The following environment configuration keys help the management command locate the appropriate AWS resources for your cluster: | key name | description | default | @@ -138,4 +141,4 @@ Run the tests: ``` $ ./scripts/test -``` \ No newline at end of file +``` From 0aff8d34bf496a7a30ca6b4cd11c5980fde839d2 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Sat, 2 Mar 2019 15:17:23 -0500 Subject: [PATCH 12/14] Specify Python 3.6+ support Make sure the README documents the proper Python support. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a96685e..cc05d7a 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ the appropriate AWS resources for your cluster: ## Developing Local development is managed with Python virtual environments. Make sure that -you have [Python 3.4+ and pip installed](https://www.python.org/downloads/) +you have [Python 3.6+ and pip installed](https://www.python.org/downloads/) before starting. Install the development package in a virtual environment: From 6c031a8374f87c9e56b0f73f706e5aed222ecafc Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Sat, 2 Mar 2019 15:33:16 -0500 Subject: [PATCH 13/14] Use AWS region name for Boto3 clients * Make sure ecsmanage configures Boto3 clients using the value of AWS_REGION provided by the user * Update ecsmanage.py docstrings --- ecsmanage/management/commands/ecsmanage.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py index 5fa4d69..3bbd2b4 100644 --- a/ecsmanage/management/commands/ecsmanage.py +++ b/ecsmanage/management/commands/ecsmanage.py @@ -33,9 +33,11 @@ def handle(self, *args, **options): cmd = options['cmd'] config = self.parse_config() + + aws_region = config['AWS_REGION'] - self.ecs_client = boto3.client('ecs') - self.ec2_client = boto3.client('ec2') + self.ecs_client = boto3.client('ecs', region_name=aws_region) + self.ec2_client = boto3.client('ec2', region_name=aws_region) task_def_arn = self.get_task_def(config['TASK_DEFINITION_NAME']) security_group_id = self.get_security_group( @@ -50,7 +52,6 @@ def handle(self, *args, **options): cmd) cluster_name = config['CLUSTER_NAME'] - aws_region = config['AWS_REGION'] url = ( f'https://console.aws.amazon.com/ecs/home?region={aws_region}#' @@ -125,7 +126,7 @@ def parse_response(self, response, key, idx=None): def get_task_def(self, task_def_name): """ - Get the ARN of the latest ECS task definition for the app CLI. + Get the ARN of the latest ECS task definition with the name task_def_name. """ task_def_response = self.ecs_client.list_task_definitions( familyPrefix=task_def_name, @@ -137,7 +138,8 @@ def get_task_def(self, task_def_name): def get_security_group(self, security_group_tags): """ - Get the ID of the security group to use for the app CLI. + Get the ID of the first security group with tags corresponding to + security_group_tags. """ filters = [] for tagname, tagvalue in security_group_tags.items(): @@ -153,7 +155,7 @@ def get_security_group(self, security_group_tags): def get_subnet(self, subnet_tags): """ - Get a subnet ID to use for the app CLI. + Get the ID of the first subnet with tags corresponding to subnet_tags. """ filters = [] for tagname, tagvalue in subnet_tags.items(): From cc8ecbb740bf6f9f5cc065ffeed25f2e7d369367 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Sat, 2 Mar 2019 15:34:02 -0500 Subject: [PATCH 14/14] Update setup.py to reflect Python 3.6+ requirement Make sure that the PyPi metadata reflects the correct requirements. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 832da94..9cde17c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url='https://github.com/azavea/django-ecsmanage/', author='Azavea', author_email='yourname@example.com', - python_requires='>=3.5', + python_requires='>=3.6', install_requires=[ 'Django >=1.11, <=2.1', 'boto3 >=1.9.0'