Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initialize Django module for one-off management commands #2

Merged
merged 27 commits into from
Apr 8, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
7a7d1ab
Scaffold Python package and move over source code
Feb 27, 2019
3dd7116
Add PR template and test artifacts
Feb 27, 2019
a35b00e
Adjust ecsmanage command to read configuration values
Feb 27, 2019
dbe4a70
Get ecsmanage command working
Feb 28, 2019
159e654
Make management command installable
Feb 28, 2019
a46dc8d
Initialize tests
Feb 28, 2019
b2e542c
Draft simple README
Feb 28, 2019
4096b3f
Add Travis config and drop support for py2.7 + py3.5
Feb 28, 2019
f42609d
Clean up Travis config
Feb 28, 2019
f929023
Document workarounds in .travis.yml
jeancochrane Mar 2, 2019
80c3ba5
Clean up README
jeancochrane Mar 2, 2019
0aff8d3
Specify Python 3.6+ support
jeancochrane Mar 2, 2019
6c031a8
Use AWS region name for Boto3 clients
jeancochrane Mar 2, 2019
cc8ecbb
Update setup.py to reflect Python 3.6+ requirement
jeancochrane Mar 2, 2019
1aeac5d
Make sure scripts/update can run in one pass
rbreslow Mar 13, 2019
004ebee
Satisfy linter requirements
rbreslow Mar 13, 2019
8ab47fa
Only pass networkConfiguration for awsvpc network mode
rbreslow Mar 14, 2019
0156d2d
fixup! Only pass networkConfiguration for awsvpc network mode
rbreslow Mar 14, 2019
0dfcaf7
Use setuptools_scm to manage Python package version
rbreslow Mar 14, 2019
94a7ccb
Fix README
hectcastro Apr 6, 2019
0433873
Add support for Black
hectcastro Apr 6, 2019
aa4e0d9
Relax all Black version constraints
hectcastro Apr 6, 2019
cfb5714
Publish to PyPi using azavea user
rbreslow Apr 8, 2019
df23ccb
Single self.stdout.write() call
rbreslow Apr 8, 2019
88b5dfa
Remove moto
rbreslow Apr 8, 2019
e7ac83c
Update MANIFEST.in
rbreslow Apr 8, 2019
fa21255
Remove deploys on develop because untagged scm version violates PEP 4…
rbreslow Apr 8, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add support for Black
Automatic source code formatting.
  • Loading branch information
hectcastro committed Apr 6, 2019
commit 043387337177946dd97096c572d3707ce4e2059e
4 changes: 2 additions & 2 deletions ecsmanage/apps.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@


class EcsManageConfig(AppConfig):
name = 'ecsmanage'
verbose_name = 'ECS Manage'
name = "ecsmanage"
verbose_name = "ECS Manage"
148 changes: 59 additions & 89 deletions ecsmanage/management/commands/ecsmanage.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,96 +4,91 @@


class Command(BaseCommand):
help = 'Run a one-off management command on an ECS cluster.'
help = "Run a one-off management command on an ECS cluster."

def add_arguments(self, parser):
parser.add_argument(
'-e', '--env',
"-e",
"--env",
type=str,
default='default',
default="default",
help=(
'Environment to run the task in, as defined in'
'ECSMANAGE_ENVIRONMENTS.'
)
"Environment to run the task in, as defined in"
"ECSMANAGE_ENVIRONMENTS."
),
)

parser.add_argument(
'cmd',
"cmd",
type=str,
nargs='+',
help="Command override for the ECS container (e.g. 'migrate')"
nargs="+",
help="Command override for the ECS container (e.g. 'migrate')",
)

def handle(self, *args, **options):
"""
Run the given command on the latest app CLI task definition and print
out a URL to view the status.
"""
self.env = options['env']
cmd = options['cmd']
self.env = options["env"]
cmd = options["cmd"]

config = self.parse_config()

aws_region = config['AWS_REGION']
aws_region = config["AWS_REGION"]

self.ecs_client = boto3.client('ecs', region_name=aws_region)
self.ec2_client = boto3.client('ec2', region_name=aws_region)
self.ecs_client = boto3.client("ecs", region_name=aws_region)
self.ec2_client = boto3.client("ec2", region_name=aws_region)

task_def_arn = self.get_task_def(config['TASK_DEFINITION_NAME'])
security_group_id = self.get_security_group(
config['SECURITY_GROUP_TAGS']
)
subnet_id = self.get_subnet(config['SUBNET_TAGS'])
task_def_arn = self.get_task_def(config["TASK_DEFINITION_NAME"])
security_group_id = self.get_security_group(config["SECURITY_GROUP_TAGS"])
subnet_id = self.get_subnet(config["SUBNET_TAGS"])

task_id = self.run_task(config,
task_def_arn,
security_group_id,
subnet_id,
cmd)
task_id = self.run_task(config, task_def_arn, security_group_id, subnet_id, cmd)

cluster_name = config['CLUSTER_NAME']
cluster_name = config["CLUSTER_NAME"]

url = (
f'https://console.aws.amazon.com/ecs/home?region={aws_region}#'
f'/clusters/{cluster_name}/tasks/{task_id}/details'
f"https://console.aws.amazon.com/ecs/home?region={aws_region}#"
f"/clusters/{cluster_name}/tasks/{task_id}/details"
)

self.stdout.write(self.style.SUCCESS('Task started! View here:\n'))
self.stdout.write(self.style.SUCCESS("Task started! View here:\n"))
self.stdout.write(self.style.SUCCESS(url))

def parse_config(self):
"""
Parse configuration settings for the app, checking to make sure that
they're valid.
"""
if getattr(settings, 'ECSMANAGE_ENVIRONMENTS') is None:
if getattr(settings, "ECSMANAGE_ENVIRONMENTS") is None:
raise CommandError(
'ECSMANAGE_ENVIRONMENTS was not found in the Django settings.'
"ECSMANAGE_ENVIRONMENTS was not found in the Django settings."
)

ecs_configs = settings.ECSMANAGE_ENVIRONMENTS.get(self.env, None)
if ecs_configs is None:
raise CommandError(
f'Environment "{self.env}" is not a recognized environment in '
'ECSMANAGE_ENVIRONMENTS (environments include: '
f'{settings.ECSMANAGE_ENVIRONMENTS.keys()})'
"ECSMANAGE_ENVIRONMENTS (environments include: "
f"{settings.ECSMANAGE_ENVIRONMENTS.keys()})"
)

config = {
'TASK_DEFINITION_NAME': '',
'CLUSTER_NAME': '',
'SECURITY_GROUP_TAGS': '',
'SUBNET_TAGS': '',
'LAUNCH_TYPE': 'FARGATE',
'AWS_REGION': 'us-east-1',
"TASK_DEFINITION_NAME": "",
"CLUSTER_NAME": "",
"SECURITY_GROUP_TAGS": "",
"SUBNET_TAGS": "",
"LAUNCH_TYPE": "FARGATE",
"AWS_REGION": "us-east-1",
}

for config_name, config_default in config.items():
if ecs_configs.get(config_name) is None:
if config_default == '':
if config_default == "":
raise CommandError(
f'Environment "{self.env}" is missing required config '
f'attribute {config_name}'
f"attribute {config_name}"
)
else:
config[config_name] = config_default
Expand All @@ -109,17 +104,14 @@ def parse_response(self, response, key, idx=None):
will get propagated to the end user.
"""
if not response.get(key):
msg = f'Unexpected response from ECS API: {response}'
msg = f"Unexpected response from ECS API: {response}"
raise KeyError(msg)
else:
if idx is not None:
try:
return response[key][0]
except (IndexError, TypeError):
msg = (
f"Unexpected value for '{key}' in response: "
f'{response}'
)
msg = f"Unexpected value for '{key}' in response: " f"{response}"
raise IndexError(msg)
else:
return response[key]
Expand All @@ -130,12 +122,10 @@ def get_task_def(self, task_def_name):
task_def_name.
"""
task_def_response = self.ecs_client.list_task_definitions(
familyPrefix=task_def_name,
sort='DESC',
maxResults=1
familyPrefix=task_def_name, sort="DESC", maxResults=1
)

return self.parse_response(task_def_response, 'taskDefinitionArns', 0)
return self.parse_response(task_def_response, "taskDefinitionArns", 0)

def get_security_group(self, security_group_tags):
"""
Expand All @@ -144,85 +134,65 @@ def get_security_group(self, security_group_tags):
"""
filters = []
for tagname, tagvalue in security_group_tags.items():
filters.append({
'Name': f'tag:{tagname}',
'Values': [tagvalue]
})
filters.append({"Name": f"tag:{tagname}", "Values": [tagvalue]})

sg_response = self.ec2_client.describe_security_groups(Filters=filters)

security_group = self.parse_response(sg_response, 'SecurityGroups', 0)
return security_group['GroupId']
security_group = self.parse_response(sg_response, "SecurityGroups", 0)
return security_group["GroupId"]

def get_subnet(self, subnet_tags):
"""
Get the ID of the first subnet with tags corresponding to subnet_tags.
"""
filters = []
for tagname, tagvalue in subnet_tags.items():
filters.append({
'Name': f'tag:{tagname}',
'Values': [tagvalue]
})
filters.append({"Name": f"tag:{tagname}", "Values": [tagvalue]})

subnet_response = self.ec2_client.describe_subnets(Filters=filters)

subnet = self.parse_response(subnet_response, 'Subnets', 0)
return subnet['SubnetId']
subnet = self.parse_response(subnet_response, "Subnets", 0)
return subnet["SubnetId"]

def run_task(self,
config,
task_def_arn,
security_group_id,
subnet_id,
cmd):
def run_task(self, config, task_def_arn, security_group_id, subnet_id, cmd):
"""
Run a task for a given task definition ARN using the given security
group and subnets, and return the task ID.
"""
overrides = {
'containerOverrides': [
{
'name': 'django',
'command': cmd
}
]
}
overrides = {"containerOverrides": [{"name": "django", "command": cmd}]}

task_def = self.ecs_client.describe_task_definition(
taskDefinition=task_def_arn,
)
task_def = self.ecs_client.describe_task_definition(taskDefinition=task_def_arn)

# Only the awsvpc network mode supports the networkConfiguration
# input value.
if task_def['networkMode'] == 'awsvpc':
if task_def["networkMode"] == "awsvpc":
network_configuration = {
'awsvpcConfiguration': {
'subnets': [subnet_id],
'securityGroups': [security_group_id]
"awsvpcConfiguration": {
"subnets": [subnet_id],
"securityGroups": [security_group_id],
}
}

task_response = self.ecs_client.run_task(
cluster=config['CLUSTER_NAME'],
cluster=config["CLUSTER_NAME"],
taskDefinition=task_def_arn,
overrides=overrides,
networkConfiguration=network_configuration,
count=1,
launchType=config['LAUNCH_TYPE']
launchType=config["LAUNCH_TYPE"],
)
else
else:
task_response = self.ecs_client.run_task(
cluster=config['CLUSTER_NAME'],
cluster=config["CLUSTER_NAME"],
taskDefinition=task_def_arn,
overrides=overrides,
count=1,
launchType=config['LAUNCH_TYPE']
launchType=config["LAUNCH_TYPE"],
)

task = self.parse_response(task_response, 'tasks', 0)
task = self.parse_response(task_response, "tasks", 0)

# Parse the ask ARN, since ECS doesn't return the task ID.
# Task ARNS look like: arn:aws:ecs:<region>:<aws_account_id>:task/<id>
task_id = task['taskArn'].split('/')[1]
task_id = task["taskArn"].split("/")[1]
return task_id
1 change: 1 addition & 0 deletions scripts/test
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ function run_linters() {

# Lint Python scripts.
./.venv/bin/flake8 --exclude ./*.pyc,./.venv
./.venv/bin/black --check --target-version py36
}

function run_tests() {
Expand Down
48 changes: 23 additions & 25 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,45 +1,43 @@
import os
from setuptools import find_packages, setup

with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
with open(os.path.join(os.path.dirname(__file__), "README.md")) as readme:
README = readme.read()

description = (
'Run any Django management command on an AWS Elastic Container Service'
'(ECS) cluster.'
"Run any Django management command on an AWS Elastic Container Service"
"(ECS) cluster."
)

# allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

setup(
name='django-ecsmanage',
name="django-ecsmanage",
use_scm_version=True,
packages=find_packages(),
include_package_data=True,
license='Apache License 2.0',
license="Apache License 2.0",
description=description,
long_description=README,
long_description_content_type='text/markdown',
url='https://github.com/azavea/django-ecsmanage/',
author='Azavea',
author_email='yourname@example.com',
python_requires='>=3.6',
install_requires=[
'Django >=1.11, <=2.1',
'boto3 >=1.9.0'
],
extras_require={'tests': ['moto >= 1.3.3', 'flake8 >= 3.7.7']},
setup_requires=['setuptools_scm==3.*'],
long_description_content_type="text/markdown",
url="https://github.com/azavea/django-ecsmanage/",
author="Azavea, Inc.",
author_email="systems@azavea.com",
python_requires=">=3.6",
install_requires=["Django>=1.11, <=2.1", "boto3>=1.9.0"],
extras_require={"tests": ["moto>=1.3.3",
"flake8>=3.7.7", "black==stable"]},
setup_requires=["setuptools_scm==3.*"],
classifiers=[
'Environment :: Web Environment',
'Framework :: Django',
'Intended Audience :: Developers',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Topic :: Internet :: WWW/HTTP',
'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
"Environment :: Web Environment",
"Framework :: Django",
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
],
)
2 changes: 1 addition & 1 deletion tests/settings_test.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""
Django settings for testing purposes.
"""
SECRET_KEY = 'testing!'
SECRET_KEY = "testing!"
Loading