diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..997e6b6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Overview + +_Add a brief description of what this PR does, and why it is needed._ + +Closes #XXX + +### Demo + +_Optional. Screenshots, `curl` examples, etc._ + +### Notes + +_Optional. Ancillary topics, caveats, alternative strategies that didn't work out, anything else._ + +## Testing Instructions + + * How to test this PR + * Prefer bulleted description + * Start after checking out this branch + * Include any setup required, such as bundling scripts, restarting services, etc. + * Include test case, and expected output \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0312f52 --- /dev/null +++ b/.gitignore @@ -0,0 +1,119 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +pyre/ + +# Editors +.vscode diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2929cc6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,45 @@ +language: python +cache: pip + +env: + global: + - PYTHONPATH="./tests/" + - DJANGO_SETTINGS_MODULE="settings_test" + +matrix: + include: + - python: 3.6 + # Workaround for Python 3.7 support. See: + # https://github.com/travis-ci/travis-ci/issues/9815 + - python: 3.7 + dist: xenial + sudo: true + +# Workaround for testing Boto3 on Travis. Taken from: +# https://github.com/azavea/django-amazon-ses/blob/57ed8619435194742308c6f15cbf9c91b231b88a/.travis.yml#L17-L19 +before_install: sudo rm /etc/boto.cfg + +install: pip install .[tests] + +script: django-admin test --noinput + +deploy: + - provider: pypi + user: django-ecsmanage + skip_existing: true + server: https://test.pypi.org/legacy/ + password: + secure: 0x32eNc+iKoYxM+nvMEEQJIoPHq3ehNqVUyKMT6crntYBBC8xgD2NHwsF0HCtfgm/NitWFlZLwtPhpb8dxTW//hh5gafkLt4/MALHFYBeRYm0AihCfTkjiuC/FCkpQ9K7X9dVO5qQN5gi5PQvkFW13NrKWOUCMTr2/jE8mLaRS3exCKUUamsd/S6BxgXY7LHm50ueDFDTYSkZrqKwIHnwwpMLcLvZZxcRJXZxpK567e8KTpcBelG+/ecY+Cn/+L0JJ29p6CGWCuW+vaIeYaeHAU6XZZ/cqNuycYlFTX+CpKSlajQgFQTMC1O1L6OVX73gSpZ2jUSd5dSimDmxgttsoo+kCxPA/hY9cUfXUpS1c52xVv9ZXZgweIV0g/EXcKLe1SIal8wy6fTE5DFzyZsZutE6ipksZjgrsFM26IHPflsf/caM/n3nDqoeXW5FtA8t4wpsemYWRFnYhCbWUWpiqAweKzzK/+3AJKEf55d4Zk7TokEjg5bZdBLroI0zy4MM6clxzujZNpn8FgTtlKhN9sDtHU5AvSe0+BUJRHJQVVSbc0wk5L+03dYarPlhShIWISSjoLFgDJhJPCZLgWD1OTetv0IekxU38AJnyqUi5XvUUbtOOiyNyU6xCFVp2XMQdRwMv41hYvEFbY4Su1B3AF2vSJCvK1HQehvzAMt4JE= + on: + distributions: sdist bdist_wheel + repo: azavea/django-ecsmanage + branch: develop + - provider: pypi + user: django-ecsmanage + skip_existing: true + password: + secure: 0x32eNc+iKoYxM+nvMEEQJIoPHq3ehNqVUyKMT6crntYBBC8xgD2NHwsF0HCtfgm/NitWFlZLwtPhpb8dxTW//hh5gafkLt4/MALHFYBeRYm0AihCfTkjiuC/FCkpQ9K7X9dVO5qQN5gi5PQvkFW13NrKWOUCMTr2/jE8mLaRS3exCKUUamsd/S6BxgXY7LHm50ueDFDTYSkZrqKwIHnwwpMLcLvZZxcRJXZxpK567e8KTpcBelG+/ecY+Cn/+L0JJ29p6CGWCuW+vaIeYaeHAU6XZZ/cqNuycYlFTX+CpKSlajQgFQTMC1O1L6OVX73gSpZ2jUSd5dSimDmxgttsoo+kCxPA/hY9cUfXUpS1c52xVv9ZXZgweIV0g/EXcKLe1SIal8wy6fTE5DFzyZsZutE6ipksZjgrsFM26IHPflsf/caM/n3nDqoeXW5FtA8t4wpsemYWRFnYhCbWUWpiqAweKzzK/+3AJKEf55d4Zk7TokEjg5bZdBLroI0zy4MM6clxzujZNpn8FgTtlKhN9sDtHU5AvSe0+BUJRHJQVVSbc0wk5L+03dYarPlhShIWISSjoLFgDJhJPCZLgWD1OTetv0IekxU38AJnyqUi5XvUUbtOOiyNyU6xCFVp2XMQdRwMv41hYvEFbY4Su1B3AF2vSJCvK1HQehvzAMt4JE= + on: + distributions: sdist bdist_wheel + repo: azavea/django-ecsmanage + branch: master diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..705e745 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Azavea Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..74215c3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..cc05d7a --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +# django-ecsmanage + +A Django app that provides a management command allowing you to run any +other management command on an AWS Elastic Container Service +(ECS) cluster. + +With `django-ecsmanage`, you can easily run migrations and other one-off tasks +on a remote cluster from the command line: + +``` +$ django-admin ecsmanage migrate +``` + +## Contents + +- [Installation](#installation) +- [Configuration](#configuration) + - [Environments](#environments) + - [AWS Resources](#aws-resources) +- [Developing](#developing) + +## Installation + +Install from PyPi using pip: + +``` +$ pip install django-ecsmanage +``` + +Update `INSTALLED_APPS` in your Django settings to install the app: + +```python +INSTALLED_APPS = ( + ... + 'ecsmanage', +) +``` + +## Configuration + +Settings for the management command are kept in a single configuration +dictionary in your Django settings named `ECSMANAGE_ENVIRONMENTS`. Each entry in +`ECSMANAGE_ENVIRONMENTS` should be a key-value pair corresponding to a +named environment (like `default` or `production`) and a set of AWS resources +associated with that environment. For example: + +```python +ECSMANAGE_ENVIRONMENTS = { + 'default': { + 'TASK_DEFINITION_NAME': 'StagingAppCLI', + 'CLUSTER_NAME': 'ecsStagingCluster', + 'LAUNCH_TYPE': 'FARGATE', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgAppEcsService', + 'Environment': 'Staging', + 'Project': 'ProjectName' + }, + 'SUBNET_TAGS': { + 'Name': 'PrivateSubnet', + 'Environment': 'Staging', + 'Project': 'ProjectName' + }, + 'AWS_REGION': 'us-east-1', + }, +} +``` + +This configuration defines a single environment, named `default`, with +associated AWS ECS resources. + +### Environments + +The key name for an environment can be any string. You can use this name +with the `--env` flag when running the command to run a command on a +different environment. Take this `ECSMANAGE_ENVIRONMENTS` configuration +as an example: + +```python +ECSMANAGE_ENVIRONMENTS = { + 'default': { + 'TASK_DEFINITION_NAME': 'StagingAppCLI', + 'CLUSTER_NAME': 'ecsStagingCluster', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgStagingAppEcsService', + }, + 'SUBNET_TAGS': { + 'Name': 'StagingPrivateSubnet', + }, + }, + 'production': { + 'TASK_DEFINITION_NAME': 'ProductionAppCLI', + 'CLUSTER_NAME': 'ecsProductionCluster', + 'SECURITY_GROUP_TAGS': { + 'Name': 'sgProductionAppEcsService', + }, + 'SUBNET_TAGS': { + 'Name': 'ProductionPrivateSubnet', + }, + }, +} +``` + +This configuration defines two environments, `default` and `production`. +Using the above settings, you could run production migrations with the +following command: + +``` +$ django-admin ecsmanage --env production migrate +``` + +If the `--env` argument is not present, the command will default to the +environment named `default`. + +### AWS Resources + +The following environment configuration keys help the management command locate +the appropriate AWS resources for your cluster: + +| key name | description | default | +| -------- | ----------- | ------- | +| `TASK_DEFINITION_NAME` | The name of your ECS task definition. The command will automatically retrieve the latest definition. | | +| `CLUSTER_NAME` | The name of your ECS cluster. | | +| `SECURITY_GROUP_TAGS` | A dictionary of tags to use to identify a security group for your task. | | +| `SUBNET_TAGS` | A dictionary of tags to use to identify a subnet for your task. | | +| `LAUNCH_TYPE` | The ECS launch type for your task. | `FARGATE` | +| `AWS_REGION` | The AWS region to run your task. | `us-east-1` | + +## Developing + +Local development is managed with Python virtual environments. Make sure that +you have [Python 3.6+ and pip installed](https://www.python.org/downloads/) +before starting. + +Install the development package in a virtual environment: + +``` +$ ./scripts/update +``` + +Run the tests: + +``` +$ ./scripts/test +``` diff --git a/ecsmanage/__init__.py b/ecsmanage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/apps.py b/ecsmanage/apps.py new file mode 100644 index 0000000..d95ac9e --- /dev/null +++ b/ecsmanage/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class EcsManageConfig(AppConfig): + name = 'ecsmanage' + verbose_name = 'ECS Manage' diff --git a/ecsmanage/management/__init__.py b/ecsmanage/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/management/commands/__init__.py b/ecsmanage/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ecsmanage/management/commands/ecsmanage.py b/ecsmanage/management/commands/ecsmanage.py new file mode 100644 index 0000000..3bbd2b4 --- /dev/null +++ b/ecsmanage/management/commands/ecsmanage.py @@ -0,0 +1,212 @@ +import boto3 +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + + +class Command(BaseCommand): + help = 'Run a one-off management command on an ECS cluster.' + + def add_arguments(self, parser): + parser.add_argument( + '-e', '--env', + type=str, + default='default', + help=( + 'Environment to run the task in, as defined in' + 'ECSMANAGE_ENVIRONMENTS.' + ) + ) + + parser.add_argument( + 'cmd', + type=str, + nargs='+', + help="Command override for the ECS container (e.g. 'migrate')" + ) + + def handle(self, *args, **options): + """ + Run the given command on the latest app CLI task definition and print + out a URL to view the status. + """ + self.env = options['env'] + cmd = options['cmd'] + + config = self.parse_config() + + aws_region = config['AWS_REGION'] + + self.ecs_client = boto3.client('ecs', region_name=aws_region) + self.ec2_client = boto3.client('ec2', region_name=aws_region) + + task_def_arn = self.get_task_def(config['TASK_DEFINITION_NAME']) + security_group_id = self.get_security_group( + config['SECURITY_GROUP_TAGS'] + ) + subnet_id = self.get_subnet(config['SUBNET_TAGS']) + + task_id = self.run_task(config, + task_def_arn, + security_group_id, + subnet_id, + cmd) + + cluster_name = config['CLUSTER_NAME'] + + url = ( + f'https://console.aws.amazon.com/ecs/home?region={aws_region}#' + f'/clusters/{cluster_name}/tasks/{task_id}/details' + ) + + self.stdout.write(self.style.SUCCESS('Task started! View here:\n')) + self.stdout.write(self.style.SUCCESS(url)) + + def parse_config(self): + """ + Parse configuration settings for the app, checking to make sure that + they're valid. + """ + if getattr(settings, 'ECSMANAGE_ENVIRONMENTS') is None: + raise CommandError( + 'ECSMANAGE_ENVIRONMENTS was not found in the Django settings.' + ) + + ecs_configs = settings.ECSMANAGE_ENVIRONMENTS.get(self.env, None) + if ecs_configs is None: + raise CommandError( + f'Environment "{self.env}" is not a recognized environment in ' + 'ECSMANAGE_ENVIRONMENTS (environments include: ' + f'{settings.ECSMANAGE_ENVIRONMENTS.keys()})' + ) + + config = { + 'TASK_DEFINITION_NAME': '', + 'CLUSTER_NAME': '', + 'SECURITY_GROUP_TAGS': '', + 'SUBNET_TAGS': '', + 'LAUNCH_TYPE': 'FARGATE', + 'AWS_REGION': 'us-east-1', + } + + for config_name, config_default in config.items(): + if ecs_configs.get(config_name) is None: + if config_default == '': + raise CommandError( + f'Environment "{self.env}" is missing required config ' + f'attribute {config_name}' + ) + else: + config[config_name] = config_default + else: + config[config_name] = ecs_configs[config_name] + + return config + + def parse_response(self, response, key, idx=None): + """ + Perform a key-value lookup on a response from the AWS API, wrapping it + in error handling such that if the lookup fails the response body + will get propagated to the end user. + """ + if not response.get(key): + msg = f'Unexpected response from ECS API: {response}' + raise KeyError(msg) + else: + if idx is not None: + try: + return response[key][0] + except (IndexError, TypeError): + msg = ( + f"Unexpected value for '{key}' in response: " + f'{response}' + ) + raise IndexError(msg) + else: + return response[key] + + def get_task_def(self, task_def_name): + """ + Get the ARN of the latest ECS task definition with the name task_def_name. + """ + task_def_response = self.ecs_client.list_task_definitions( + familyPrefix=task_def_name, + sort='DESC', + maxResults=1 + ) + + return self.parse_response(task_def_response, 'taskDefinitionArns', 0) + + def get_security_group(self, security_group_tags): + """ + Get the ID of the first security group with tags corresponding to + security_group_tags. + """ + filters = [] + for tagname, tagvalue in security_group_tags.items(): + filters.append({ + 'Name': f'tag:{tagname}', + 'Values': [tagvalue] + }) + + sg_response = self.ec2_client.describe_security_groups(Filters=filters) + + security_group = self.parse_response(sg_response, 'SecurityGroups', 0) + return security_group['GroupId'] + + def get_subnet(self, subnet_tags): + """ + Get the ID of the first subnet with tags corresponding to subnet_tags. + """ + filters = [] + for tagname, tagvalue in subnet_tags.items(): + filters.append({ + 'Name': f'tag:{tagname}', + 'Values': [tagvalue] + }) + + subnet_response = self.ec2_client.describe_subnets(Filters=filters) + + subnet = self.parse_response(subnet_response, 'Subnets', 0) + return subnet['SubnetId'] + + def run_task(self, + config, + task_def_arn, + security_group_id, + subnet_id, + cmd): + """ + Run a task for a given task definition ARN using the given security + group and subnets, and return the task ID. + """ + overrides = { + 'containerOverrides': [ + { + 'name': 'django', + 'command': cmd + } + ] + } + + network_configuration = { + 'awsvpcConfiguration': { + 'subnets': [subnet_id], + 'securityGroups': [security_group_id] + } + } + + task_response = self.ecs_client.run_task( + cluster=config['CLUSTER_NAME'], + taskDefinition=task_def_arn, + overrides=overrides, + networkConfiguration=network_configuration, + count=1, + launchType=config['LAUNCH_TYPE'] + ) + + task = self.parse_response(task_response, 'tasks', 0) + + # Parse the ask ARN, since ECS doesn't return the task ID. + # Task ARNS look like: arn:aws:ecs:::task/ + task_id = task['taskArn'].split('/')[1] + return task_id diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000..9cd87c6 --- /dev/null +++ b/scripts/test @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECSMANAGE_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") [OPTIONS] + +Run tests for the django-ecsmanage app. + +Options: + --help Display help text. + --lint Run shell and Python linters. + --app Run app tests. +" +} + +function run_linters() { + # Lint Bash scripts. + if command -v shellcheck > /dev/null; then + shellcheck scripts/* + fi + + # Lint Python scripts. + ./.venv/bin/flake8 --exclude ./*.pyc,./.venv +} + +function run_tests() { + PYTHONPATH="./tests/" DJANGO_SETTINGS_MODULE="settings_test" \ + ./.venv/bin/django-admin test --noinput +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + elif [ "${1:-}" = "--lint" ]; then + run_linters + elif [ "${1:-}" = "--app" ]; then + run_tests + else + run_linters + run_tests + fi +fi diff --git a/scripts/update b/scripts/update new file mode 100755 index 0000000..b778c9c --- /dev/null +++ b/scripts/update @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +if [[ -n "${ECSMANAGE_DEBUG}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") + +Install the package for testing. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + if ! [ -x "$(command -v python3)" ]; then + echo "Error: python3 is not installed." + exit 1 + elif ! [ -x "$(command -v pip3)" ]; then + echo "Error: pip3 is not installed." + exit 1 + else + if ! [ -d ".venv" ]; then + python3 -m venv .venv + else + ./.venv/bin/pip3 install -e ".[tests]" + fi + fi + fi +fi diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9cde17c --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +import os +from setuptools import find_packages, setup + +with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme: + README = readme.read() + +description = ( + 'Run any Django management command on an AWS Elastic Container Service' + '(ECS) cluster.' +) + +# allow setup.py to be run from any path +os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + +setup( + name='django-ecsmanage', + version='0.0.0', + packages=find_packages(), + include_package_data=True, + license='Apache License 2.0', + description=description, + long_description=README, + long_description_content_type='text/markdown', + url='https://github.com/azavea/django-ecsmanage/', + author='Azavea', + author_email='yourname@example.com', + python_requires='>=3.6', + install_requires=[ + 'Django >=1.11, <=2.1', + 'boto3 >=1.9.0' + ], + extras_require={'tests': ['moto >= 1.3.3', 'flake8 >= 3.7.7']}, + classifiers=[ + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + ], +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/settings_test.py b/tests/settings_test.py new file mode 100644 index 0000000..1b5928f --- /dev/null +++ b/tests/settings_test.py @@ -0,0 +1,4 @@ +""" +Django settings for testing purposes. +""" +SECRET_KEY = 'testing!' diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..6280722 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,93 @@ +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import SimpleTestCase + + +class ConfigurationTestCase(SimpleTestCase): + """ + Test the configuration of settings for the management command. + """ + def test_failure_when_no_settings(self): + """ + Test that the command throws an error when the ECSMANAGE_ENVIRONMENTS + setting does not exist. + """ + with self.assertRaises(CommandError): + call_command('ecsmanage', 'help') + + def test_failure_when_missing_environment(self): + """ + Test that the command throws an error when the environment passed to + the CLI does not exist in ECSMANAGE_ENVIRONMENTS. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': {}, + 'production': {} + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='foobar') + + def test_failure_when_no_task_def_name(self): + """ + Test that the command throws an error when the configuration is missing + a task definition name. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'CLUSTER_NAME': 'foo', + 'SECURITY_GROUP_TAGS': {}, + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_cluster_name(self): + """ + Test that the command throws an error when the configuration is missing + a cluster name. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'SECURITY_GROUP_TAGS': {}, + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_security_group_tags(self): + """ + Test that the command throws an error when the configuration is missing + security group tags. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'CLUSTER_NAME': 'bar', + 'SUBNET_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging') + + def test_failure_when_no_subnet_tags(self): + """ + Test that the command throws an error when the configuration is missing + subnet tags. + """ + ECSMANAGE_ENVIRONMENTS = { + 'staging': { + 'TASK_DEFINITION_NAME': 'foo', + 'CLUSTER_NAME': 'bar', + 'SECURITY_GROUP_TAGS': {}, + }, + } + with self.assertRaises(CommandError): + with self.settings(ECSMANAGE_ENVIRONMENTS=ECSMANAGE_ENVIRONMENTS): + call_command('ecsmanage', 'help', env='staging')