From ae39a697260fe780d03e4bcc5960739bb01ea281 Mon Sep 17 00:00:00 2001 From: "Ernest W. Durbin III" Date: Wed, 27 Nov 2013 18:15:34 -0500 Subject: [PATCH] fixup the development environment for pypi --- Vagrantfile | 14 ++- provisioning/salt/minion/web/pypi_dev | 8 ++ .../roots/pillar/pypi-deploys/pypi-dev.sls | 24 +++++ .../salt/roots/pillar/secrets/pypi-dev.sls | 27 +++++ provisioning/salt/roots/pillar/top.sls | 11 +- provisioning/salt/roots/salt/pypi/dev-db.sls | 101 +++++++++++------- 6 files changed, 144 insertions(+), 41 deletions(-) create mode 100644 provisioning/salt/minion/web/pypi_dev create mode 100644 provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls create mode 100644 provisioning/salt/roots/pillar/secrets/pypi-dev.sls diff --git a/Vagrantfile b/Vagrantfile index 2194d6a..b612b66 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,8 +12,20 @@ Vagrant.configure("2") do |config| config.vm.synced_folder "provisioning/salt/roots/", "/srv/" + if ENV['VAGRANT_PYPI_DEV'] == '1' + config.vm.define "pypi_dev" do |pypi_dev| - unless ENV['VAGRANT_SKIP_PYPI'] == '1' + pypi_dev.vm.network "private_network", ip: "192.168.57.9" + + pypi_dev.vm.provision :salt do |s| + s.verbose = true + s.minion_config = "provisioning/salt/minion/web/pypi_dev" + s.run_highstate = true + end + end + end + + if ENV['VAGRANT_PYPI'] == '1' config.vm.define "pypi" do |pypi| pypi.vm.network "private_network", ip: "192.168.57.10" diff --git a/provisioning/salt/minion/web/pypi_dev b/provisioning/salt/minion/web/pypi_dev new file mode 100644 index 0000000..8fa7376 --- /dev/null +++ b/provisioning/salt/minion/web/pypi_dev @@ -0,0 +1,8 @@ +file_client: local +file_roots: + base: + - /srv/salt +grains: + roles: + - develop + - pypi diff --git a/provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls b/provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls new file mode 100644 index 0000000..cb36354 --- /dev/null +++ b/provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls @@ -0,0 +1,24 @@ + +pypi-deploy-devpypi: + name: devpypi + + user: devpypi + group: devpypi + user_uid: 6666 + group_gid: 6666 + + path: /opt/devpypi + data_mount: /data/devpypi + data_device: + type: local + uri: None + + mailhost: localhost:25 + + https_only: True + server_names: + - 192.168.57.9 + url: http://192.168.57.9 + + uwsgi: + processes: 2 diff --git a/provisioning/salt/roots/pillar/secrets/pypi-dev.sls b/provisioning/salt/roots/pillar/secrets/pypi-dev.sls new file mode 100644 index 0000000..9beda0f --- /dev/null +++ b/provisioning/salt/roots/pillar/secrets/pypi-dev.sls @@ -0,0 +1,27 @@ + +secrets-pypi-deploy-devpypi: + postgresql: + host: 127.0.0.1 + database: testpypi + user: testpypi + password: testpypi + webui: + cheescake_password: secret + reset_secret: secret + logging: + mailhost: mail.python.org + fromaddr: foo@bar.com + toaddrs: fiz@bar.com + sentry: + dsn: '' + fastly: + api_key: '' + service_id: '' + pubkey: | + -----BEGIN PUBLIC KEY----- + publiclolnope + -----END PUBLIC KEY----- + privkey: | + -----BEGIN DSA PRIVATE KEY----- + privatelolnope + -----END DSA PRIVATE KEY----- diff --git a/provisioning/salt/roots/pillar/top.sls b/provisioning/salt/roots/pillar/top.sls index 3d5ec85..f441f45 100644 --- a/provisioning/salt/roots/pillar/top.sls +++ b/provisioning/salt/roots/pillar/top.sls @@ -12,12 +12,19 @@ base: 'roles:pypi-mirror': - match: grain - pypi-mirror - 'roles:pypi': - - match: grain + + 'G@roles:pypi not G@roles:develop': + - match: compound - pypi - pypi-deploys.testpypi - secrets.testpypi + 'G@roles:pypi and G@roles:develop': + - match: compound + - pypi + - pypi-deploys.pypi-dev + - secrets.pypi-dev + 'roles:postgresql_cluster': - match: grain - postgresql.cluster diff --git a/provisioning/salt/roots/salt/pypi/dev-db.sls b/provisioning/salt/roots/salt/pypi/dev-db.sls index 22516d1..6504653 100644 --- a/provisioning/salt/roots/salt/pypi/dev-db.sls +++ b/provisioning/salt/roots/salt/pypi/dev-db.sls @@ -1,3 +1,10 @@ +{% set deploys = {} %} +{% for k,v in pillar.items() %} + {% if k.startswith('pypi-deploy-') %} + {% do deploys.update({k: v}) %} + {% endif %} +{% endfor %} + postgresql: pkg.installed @@ -16,69 +23,87 @@ pypi_postgres_pg_hba: - pkg: postgresql93-server - cmd: postgresql93-server -pypi_dev_postgres_user: +{% for key, config in deploys.items() %} +{% set secrets = salt['pillar.get']("secrets-"+key) %} + +{{ config['name'] }}_dev_postgres_user: postgres_user.present: - - name: pypi - - password: pypi + - name: {{ secrets['postgresql']['user'] }} + - password: {{ secrets['postgresql']['password'] }} - user: postgres - require: - service: postgresql-9.3 - pkg: postgresql -pypi_postgres_database: +{{ config['name'] }}_postgres_database: postgres_database.present: - - name: pypi - - owner: pypi + - name: {{ secrets['postgresql']['database'] }} + - owner: {{ secrets['postgresql']['user'] }} - require: - - postgres_user: pypi_dev_postgres_user + - postgres_user: {{ config['name'] }}_dev_postgres_user -pypi_postgres_citext: +{{ config['name'] }}_postgres_citext: cmd.wait: - - name: 'psql pypi -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' + - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' - user: postgres - require: - - postgres_database: pypi_postgres_database - - postgres_user: pypi_dev_postgres_user + - postgres_database: {{ config['name'] }}_postgres_database + - postgres_user: {{ config['name'] }}_dev_postgres_user - pkg: pypi_postgres_contrib - watch: - - postgres_database: pypi_postgres_database - - postgres_user: pypi_dev_postgres_user + - postgres_database: {{ config['name'] }}_postgres_database + - postgres_user: {{ config['name'] }}_dev_postgres_user - pkg: pypi_postgres_contrib -pypi_postgres_plpgsql: +{{ config['name'] }}_postgres_plpgsql: cmd.wait: - - name: 'psql pypi -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' + - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"' - user: postgres - require: - - postgres_database: pypi_postgres_database - - postgres_user: pypi_dev_postgres_user + - postgres_database: {{ config['name'] }}_postgres_database + - postgres_user: {{ config['name'] }}_dev_postgres_user - watch: - - postgres_database: pypi_postgres_database - - postgres_user: pypi_dev_postgres_user + - postgres_database: {{ config['name'] }}_postgres_database + - postgres_user: {{ config['name'] }}_dev_postgres_user -pypi_schema_load: +{{ config['path'] }}/.pgpass: + file.managed: + - contents: "{{ secrets['postgresql']['host'] }}:*:{{ secrets['postgresql']['database'] }}:{{ secrets['postgresql']['user'] }}:{{ secrets['postgresql']['password'] }}" + - user: {{ config['user'] }} + - group: {{ config['group'] }} + - mode: 600 + - require: + - user: {{ config['user'] }} + - group: {{ config['group'] }} + +{{ config['name'] }}_schema_load: cmd.wait: - - name: 'psql pypi -f /opt/pypi/src/pkgbase_schema.sql' - - user: pypi - - cwd: /opt/pypi/src + - name: 'psql {{ secrets['postgresql']['database'] }} -U {{ secrets['postgresql']['user'] }} -h {{ secrets['postgresql']['host'] }} -f {{ config['path'] }}/src/pkgbase_schema.sql' + - user: {{ config['user'] }} + - cwd: {{ config['path'] }} + - env: + PGPASSWORD: {{ secrets['postgresql']['password'] }} - require: - - postgres_database: pypi_postgres_database - - postgres_user: pypi_dev_postgres_user - - cmd: pypi_postgres_plpgsql - - cmd: pypi_postgres_citext - - hg: pypi-source + - postgres_database: {{ config['name'] }}_postgres_database + - postgres_user: {{ config['name'] }}_dev_postgres_user + - cmd: {{ config['name'] }}_postgres_plpgsql + - cmd: {{ config['name'] }}_postgres_citext + - hg: {{ config['name'] }}-source + - file: {{ config['path'] }}/.pgpass - watch: - - postgres_database: pypi_postgres_database + - postgres_database: {{ config['name'] }}_postgres_database -pypi_sample_data_load: +{{ config['name'] }}_sample_data_load: cmd.wait: - - name: '/opt/pypi/env/bin/python /opt/pypi/src/tools/demodata.py' - - user: pypi - - cwd: /opt/pypi/src + - name: '{{ config['path'] }}/env/bin/python {{ config['path'] }}/src/tools/demodata.py' + - user: {{ config['user'] }} + - cwd: {{ config['path'] }}/src - require: - - cmd: pypi_schema_load - - virtualenv: /opt/pypi/env - - file: /opt/pypi/src/config.ini - - file: /data/pypi + - cmd: {{ config['name'] }}_schema_load + - virtualenv: {{ config['path'] }}/env + - file: {{ config['path'] }}/src/config.ini + - file: {{ config['data_mount'] }} - watch: - - postgres_database: pypi_postgres_database + - postgres_database: {{ config['name'] }}_postgres_database + +{% endfor %}