fixup the development environment for pypi

pypi · Nov 27, 2013 · ae39a69 · ae39a69
1 parent 53230cf
commit ae39a69
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 41 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -12,8 +12,20 @@ Vagrant.configure("2") do |config|
 
   config.vm.synced_folder "provisioning/salt/roots/", "/srv/"
 
+  if ENV['VAGRANT_PYPI_DEV'] == '1'
+    config.vm.define "pypi_dev" do |pypi_dev|
 
-  unless ENV['VAGRANT_SKIP_PYPI'] == '1'
+      pypi_dev.vm.network "private_network", ip: "192.168.57.9"
+
+      pypi_dev.vm.provision :salt do |s|
+        s.verbose = true
+        s.minion_config = "provisioning/salt/minion/web/pypi_dev"
+        s.run_highstate = true
+      end
+    end
+  end
+
+  if ENV['VAGRANT_PYPI'] == '1'
     config.vm.define "pypi" do |pypi|
 
       pypi.vm.network "private_network", ip: "192.168.57.10"

diff --git a/provisioning/salt/minion/web/pypi_dev b/provisioning/salt/minion/web/pypi_dev
@@ -0,0 +1,8 @@
+file_client: local
+file_roots:
+  base:
+    - /srv/salt
+grains:
+  roles:
+    - develop
+    - pypi
diff --git a/provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls b/provisioning/salt/roots/pillar/pypi-deploys/pypi-dev.sls
@@ -0,0 +1,24 @@
+
+pypi-deploy-devpypi:
+  name: devpypi
+
+  user: devpypi
+  group: devpypi
+  user_uid: 6666
+  group_gid: 6666
+
+  path: /opt/devpypi
+  data_mount: /data/devpypi
+  data_device:
+    type: local
+    uri: None
+
+  mailhost: localhost:25
+
+  https_only: True
+  server_names:
+    - 192.168.57.9
+  url: http://192.168.57.9
+
+  uwsgi:
+    processes: 2
diff --git a/provisioning/salt/roots/pillar/secrets/pypi-dev.sls b/provisioning/salt/roots/pillar/secrets/pypi-dev.sls
@@ -0,0 +1,27 @@
+
+secrets-pypi-deploy-devpypi:
+  postgresql:
+    host: 127.0.0.1
+    database: testpypi
+    user: testpypi
+    password: testpypi
+  webui:
+    cheescake_password: secret
+    reset_secret: secret
+  logging:
+    mailhost: mail.python.org
+    fromaddr: foo@bar.com
+    toaddrs: fiz@bar.com
+  sentry:
+    dsn: ''
+  fastly:
+    api_key: ''
+    service_id: ''
+  pubkey: |
+    -----BEGIN PUBLIC KEY-----
+    publiclolnope
+    -----END PUBLIC KEY-----
+  privkey: |
+    -----BEGIN DSA PRIVATE KEY-----
+    privatelolnope
+    -----END DSA PRIVATE KEY-----
diff --git a/provisioning/salt/roots/pillar/top.sls b/provisioning/salt/roots/pillar/top.sls
@@ -12,12 +12,19 @@ base:
   'roles:pypi-mirror':
     - match: grain
     - pypi-mirror
-  'roles:pypi':
-    - match: grain
+
+  'G@roles:pypi not G@roles:develop':
+    - match: compound
     - pypi
     - pypi-deploys.testpypi
     - secrets.testpypi
 
+  'G@roles:pypi and G@roles:develop':
+    - match: compound
+    - pypi
+    - pypi-deploys.pypi-dev
+    - secrets.pypi-dev
+
   'roles:postgresql_cluster':
     - match: grain
     - postgresql.cluster

diff --git a/provisioning/salt/roots/salt/pypi/dev-db.sls b/provisioning/salt/roots/salt/pypi/dev-db.sls
@@ -1,3 +1,10 @@
+{% set deploys = {} %}
+{% for k,v in pillar.items() %}
+  {% if k.startswith('pypi-deploy-') %}
+    {% do deploys.update({k: v}) %}
+  {% endif %}
+{% endfor %}
+
 postgresql:
   pkg.installed
 
@@ -16,69 +23,87 @@ pypi_postgres_pg_hba:
       - pkg: postgresql93-server
       - cmd: postgresql93-server
 
-pypi_dev_postgres_user:
+{% for key, config in deploys.items() %}
+{% set secrets = salt['pillar.get']("secrets-"+key) %}
+
+{{ config['name'] }}_dev_postgres_user:
   postgres_user.present:
-    - name: pypi
-    - password: pypi
+    - name: {{ secrets['postgresql']['user'] }}
+    - password: {{ secrets['postgresql']['password'] }}
     - user: postgres
     - require:
       - service: postgresql-9.3
       - pkg: postgresql
 
-pypi_postgres_database:
+{{ config['name'] }}_postgres_database:
   postgres_database.present:
-    - name: pypi
-    - owner: pypi
+    - name: {{ secrets['postgresql']['database'] }}
+    - owner: {{ secrets['postgresql']['user'] }}
     - require:
-      - postgres_user: pypi_dev_postgres_user
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
 
-pypi_postgres_citext:
+{{ config['name'] }}_postgres_citext:
   cmd.wait:
-    - name: 'psql pypi -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"'
+    - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"'
     - user: postgres
     - require:
-      - postgres_database: pypi_postgres_database
-      - postgres_user: pypi_dev_postgres_user
+      - postgres_database: {{ config['name'] }}_postgres_database
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
       - pkg: pypi_postgres_contrib
     - watch:
-      - postgres_database: pypi_postgres_database
-      - postgres_user: pypi_dev_postgres_user
+      - postgres_database: {{ config['name'] }}_postgres_database
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
       - pkg: pypi_postgres_contrib
 
-pypi_postgres_plpgsql:
+{{ config['name'] }}_postgres_plpgsql:
   cmd.wait:
-    - name: 'psql pypi -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"'
+    - name: 'psql {{ secrets['postgresql']['database'] }} -c "CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public;"'
     - user: postgres
     - require:
-      - postgres_database: pypi_postgres_database
-      - postgres_user: pypi_dev_postgres_user
+      - postgres_database: {{ config['name'] }}_postgres_database
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
     - watch:
-      - postgres_database: pypi_postgres_database
-      - postgres_user: pypi_dev_postgres_user
+      - postgres_database: {{ config['name'] }}_postgres_database
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
 
-pypi_schema_load:
+{{ config['path'] }}/.pgpass:
+  file.managed:
+    - contents: "{{ secrets['postgresql']['host'] }}:*:{{ secrets['postgresql']['database'] }}:{{ secrets['postgresql']['user'] }}:{{ secrets['postgresql']['password'] }}"
+    - user: {{ config['user'] }}
+    - group: {{ config['group'] }}
+    - mode: 600
+    - require:
+      - user: {{ config['user'] }}
+      - group: {{ config['group'] }}
+
+{{ config['name'] }}_schema_load:
   cmd.wait:
-    - name: 'psql pypi -f /opt/pypi/src/pkgbase_schema.sql'
-    - user: pypi
-    - cwd: /opt/pypi/src
+    - name: 'psql {{ secrets['postgresql']['database'] }} -U {{ secrets['postgresql']['user'] }} -h {{ secrets['postgresql']['host'] }} -f {{ config['path'] }}/src/pkgbase_schema.sql'
+    - user: {{ config['user'] }}
+    - cwd: {{ config['path'] }}
+    - env:
+      PGPASSWORD: {{ secrets['postgresql']['password'] }}
     - require:
-      - postgres_database: pypi_postgres_database
-      - postgres_user: pypi_dev_postgres_user
-      - cmd: pypi_postgres_plpgsql
-      - cmd: pypi_postgres_citext
-      - hg: pypi-source
+      - postgres_database: {{ config['name'] }}_postgres_database
+      - postgres_user: {{ config['name'] }}_dev_postgres_user
+      - cmd: {{ config['name'] }}_postgres_plpgsql
+      - cmd: {{ config['name'] }}_postgres_citext
+      - hg: {{ config['name'] }}-source
+      - file: {{ config['path'] }}/.pgpass
     - watch:
-      - postgres_database: pypi_postgres_database
+      - postgres_database: {{ config['name'] }}_postgres_database
 
-pypi_sample_data_load:
+{{ config['name'] }}_sample_data_load:
   cmd.wait:
-    - name: '/opt/pypi/env/bin/python /opt/pypi/src/tools/demodata.py'
-    - user: pypi
-    - cwd: /opt/pypi/src
+    - name: '{{ config['path'] }}/env/bin/python {{ config['path'] }}/src/tools/demodata.py'
+    - user: {{ config['user'] }}
+    - cwd: {{ config['path'] }}/src
     - require:
-      - cmd: pypi_schema_load
-      - virtualenv: /opt/pypi/env
-      - file: /opt/pypi/src/config.ini
-      - file: /data/pypi
+      - cmd: {{ config['name'] }}_schema_load
+      - virtualenv: {{ config['path'] }}/env
+      - file: {{ config['path'] }}/src/config.ini
+      - file: {{ config['data_mount'] }}
     - watch:
-      - postgres_database: pypi_postgres_database
+      - postgres_database: {{ config['name'] }}_postgres_database
+
+{% endfor %}