add riemann/graphite monitoring server

pypi · Nov 29, 2013 · 9486471 · 9486471
1 parent 2695dad
commit 9486471
Show file tree

Hide file tree

Showing 16 changed files with 463 additions and 0 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -25,6 +25,21 @@ Vagrant.configure("2") do |config|
     end
   end
 
+  if ENV['VAGRANT_MONITORING'] == '1'
+    config.vm.define "monitoring_server" do |monitoring_server|
+
+      monitoring_server.vm.hostname = "pypi-monitoring"
+      monitoring_server.vm.network "private_network", ip: "192.168.57.200"
+      monitoring_server.vm.network "private_network", ip: "172.16.57.200"
+
+      monitoring_server.vm.provision :salt do |s|
+        s.verbose = true
+        s.minion_config = "provisioning/salt/minion/monitoring/server"
+        s.run_highstate = true
+      end
+    end
+  end
+
   if ENV['VAGRANT_PYPI'] == '1'
     config.vm.define "pypi" do |pypi|
 

diff --git a/provisioning/salt/minion/monitoring/server b/provisioning/salt/minion/monitoring/server
@@ -0,0 +1,7 @@
+file_client: local
+file_roots:
+  base:
+    - /srv/salt
+grains:
+  roles:
+    - monitoring_server
diff --git a/provisioning/salt/roots/pillar/monitoring/server/init.sls b/provisioning/salt/roots/pillar/monitoring/server/init.sls
@@ -0,0 +1,45 @@
+{% set secrets = pillar.get('monitoring-secrets', {}) %}
+
+firewall:
+  riemann_ports:
+    port: 5555:5556
+    source: 172.16.57.0/24
+  riemann_udp_ports:
+    port: 5556
+    source: 172.16.57.0/24
+  riemann_graphite:
+    port: 2002
+    source: 172.16.57.0/24
+  graphite_ports:
+    port: 2003:2004
+    source: 172.16.57.0/24
+  graphite_query_port:
+    port: 7002
+    source: 172.16.57.0/24
+  http:
+    port: 80
+  https:
+    port: 443
+
+riemann:
+  host: 0.0.0.0
+  graphite_host: 0.0.0.0
+  graphite_port: 2002
+  graphite_downstream_host: 127.0.0.1
+  graphite_downstream_port: 2003
+
+graphite:
+  secret_key: {{ secrets.get('graphite', {}).get('secret_key', 'deadbeef') }}
+  allowed_hosts: '*'
+  sqlite3_path: /var/lib/graphite-web/graphite.db
+  https_only: true
+  server_names:
+    - 172.16.57.200
+
+carbon:
+  line_receiver_interface: 0.0.0.0
+  line_reciver_port: 2003
+  pickle_receiver_interface: 0.0.0.0
+  pickle_receiver_port: 2004
+  cache_query_interface: 0.0.0.0
+  cache_query_port: 7002
diff --git a/provisioning/salt/roots/pillar/secrets/monitoring.sls b/provisioning/salt/roots/pillar/secrets/monitoring.sls
@@ -0,0 +1,3 @@
+monitoring-secrets:
+  graphite:
+    secret_key: foobarbaz
diff --git a/provisioning/salt/roots/pillar/top.sls b/provisioning/salt/roots/pillar/top.sls
@@ -40,3 +40,7 @@ base:
   'roles:gluster_node':
     - match: grain
     - glusterfs.server
+
+  'roles:monitoring_server':
+    - match: grain
+    - monitoring.server
diff --git a/provisioning/salt/roots/salt/monitoring/server/carbon.sls b/provisioning/salt/roots/salt/monitoring/server/carbon.sls
@@ -0,0 +1,30 @@
+
+{% set carbon_config = pillar.get('carbon', {}) %}
+{% set graphite_config = pillar.get('graphite', {}) %}
+
+graphite-pkgs:
+  pkg.installed:
+    - pkgs:
+      - graphite-web
+      - python-carbon
+      - httpd
+
+/etc/carbon/carbon.conf:
+  file.managed:
+    - source: salt://monitoring/server/config/carbon.conf.jinja
+    - template: jinja
+    - context:
+      line_receiver_interface: {{ carbon_config.get('line_receiver_interface', '0.0.0.0') }}
+      line_reciver_port: {{ carbon_config.get('line_reciver_port', 2003) }}
+      pickle_receiver_interface: {{ carbon_config.get('pickle_receiver_interface', '0.0.0.0') }}
+      pickle_receiver_port: {{ carbon_config.get('pickle_receiver_port', 2004) }}
+      cache_query_interface: {{ carbon_config.get('cache_query_interface', '0.0.0.0') }}
+      cache_query_port: {{ carbon_config.get('cache_query_port', 7002) }}
+
+carbon-cache:
+  service:
+    - running
+    - enable: True
+    - restart: True
+    - watch:
+      - file: /etc/carbon/carbon.conf
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/carbon.conf.jinja b/provisioning/salt/roots/salt/monitoring/server/config/carbon.conf.jinja
@@ -0,0 +1,100 @@
+[cache]
+STORAGE_DIR    = /var/lib/carbon/
+LOCAL_DATA_DIR = /var/lib/carbon/whisper/
+WHITELISTS_DIR = /var/lib/carbon/lists/
+CONF_DIR       = /etc/carbon/
+LOG_DIR        = /var/log/carbon/
+PID_DIR        = /var/run/
+
+ENABLE_LOGROTATION = True
+USER = carbon
+MAX_CACHE_SIZE = inf
+MAX_UPDATES_PER_SECOND = 500
+MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000
+MAX_CREATES_PER_MINUTE = 50
+
+LINE_RECEIVER_INTERFACE = {{ line_receiver_interface }}
+LINE_RECEIVER_PORT = {{ line_reciver_port }}
+
+PICKLE_RECEIVER_INTERFACE = {{ pickle_receiver_interface }}
+PICKLE_RECEIVER_PORT = {{ pickle_receiver_port }}
+
+LOG_LISTENER_CONNECTIONS = True
+USE_INSECURE_UNPICKLER = False
+
+CACHE_QUERY_INTERFACE = {{ cache_query_interface }}
+CACHE_QUERY_PORT = {{ cache_query_port }}
+
+USE_FLOW_CONTROL = True
+
+LOG_UPDATES = False
+LOG_CACHE_HITS = False
+LOG_CACHE_QUEUE_SORTS = True
+
+CACHE_WRITE_STRATEGY = sorted
+
+WHISPER_AUTOFLUSH = False
+
+# WHISPER_SPARSE_CREATE = False
+WHISPER_FALLOCATE_CREATE = True
+# WHISPER_LOCK_WRITES = False
+# USE_WHITELIST = False
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60
+
+[relay]
+USER = carbon
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2013
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2014
+
+LOG_LISTENER_CONNECTIONS = True
+
+#RELAY_METHOD = rules
+#RELAY_METHOD = consistent-hashing
+#RELAY_METHOD = aggregated-consistent-hashing
+RELAY_METHOD = rules
+
+REPLICATION_FACTOR = 1
+
+DESTINATIONS = 127.0.0.1:2004
+
+MAX_DATAPOINTS_PER_MESSAGE = 500
+MAX_QUEUE_SIZE = 10000
+
+USE_FLOW_CONTROL = True
+
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60
+
+[aggregator]
+USER = carbon
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2023
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2024
+
+LOG_LISTENER_CONNECTIONS = True
+
+FORWARD_ALL = True
+
+DESTINATIONS = 127.0.0.1:2004
+
+REPLICATION_FACTOR = 1
+
+MAX_QUEUE_SIZE = 10000
+
+USE_FLOW_CONTROL = True
+
+MAX_DATAPOINTS_PER_MESSAGE = 500
+
+MAX_AGGREGATION_INTERVALS = 5
+
+# WRITE_BACK_FREQUENCY = 0
+
+# USE_WHITELIST = False
+
+# CARBON_METRIC_PREFIX = carbon
+# CARBON_METRIC_INTERVAL = 60
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/graphite-web-app.conf.jinja b/provisioning/salt/roots/salt/monitoring/server/config/graphite-web-app.conf.jinja
@@ -0,0 +1,20 @@
+client_max_body_size 64M;
+keepalive_timeout 5;
+
+access_log /var/log/nginx/graphite-web/access.log;
+error_log /var/log/nginx/graphite-web/error.log;
+
+root /usr/share/graphite/webapp;
+
+location /media/ {
+  root /usr/lib/python2.6/site-packages/django/contrib/admin/media;
+}
+
+try_files $uri/index.html $uri.html $uri @app;
+
+location @app {
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  proxy_set_header Host $http_host;
+  proxy_redirect off;
+  proxy_pass http://graphite_web_server;
+}
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/graphite-web-nginx.conf.jinja b/provisioning/salt/roots/salt/monitoring/server/config/graphite-web-nginx.conf.jinja
@@ -0,0 +1,23 @@
+upstream graphite_web_server {
+  server unix:/var/run/graphite-web/gunicorn.sock fail_timeout=0;
+}
+
+server {
+  listen 80;
+  server_name {{ server_names }};
+
+  {% if https_only %}
+  location / {
+    return 301 https://$server_name$request_uri;
+  }
+  {% else %}
+  include conf.d/graphite-web/app.conf;
+  {% endif %}
+}
+
+server {
+  listen 443;
+  server_name {{ server_names }};
+  include nginx.ssl.conf;
+  include conf.d/graphite-web/app.conf;
+}
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/graphite-web.ini.jinja b/provisioning/salt/roots/salt/monitoring/server/config/graphite-web.ini.jinja
@@ -0,0 +1,10 @@
+[program:{{ program_name }}]
+command = gunicorn_django --bind unix:/var/run/graphite-web/gunicorn.sock --pythonpath=/usr/lib/python2.6/site-packages/graphite --settings=settings settings.py
+autostart = true
+autorestart = true
+stopwaitsecs = 15
+stopsignal = TERM
+stdout_logfile = /var/log/{{ program_name }}/gunicorn-stdout.log
+stderr_logfile = /var/log/{{ program_name }}/gunicorn-stderr.log
+user = {{ user }}
+directory = /usr/lib/python2.6/site-packages/graphite
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/local_settings.py.jinja b/provisioning/salt/roots/salt/monitoring/server/config/local_settings.py.jinja
@@ -0,0 +1,36 @@
+SECRET_KEY = '{{ secret_key }}'
+ALLOWED_HOSTS = [ '{{ allowed_hosts }}' ]
+#TIME_ZONE = 'America/Los_Angeles'
+
+LOG_RENDERING_PERFORMANCE = True
+LOG_CACHE_PERFORMANCE = True
+LOG_METRIC_ACCESS = True
+
+DEBUG = False
+
+GRAPHITE_ROOT = '/usr/share/graphite'
+CONF_DIR = '/etc/graphite-web'
+STORAGE_DIR = '/var/lib/graphite-web'
+CONTENT_DIR = '/usr/share/graphite/webapp/content'
+#DASHBOARD_CONF = '/etc/graphite-web/dashboard.conf'
+#GRAPHTEMPLATES_CONF = '/etc/graphite-web/graphTemplates.conf'
+
+WHISPER_DIR = '/var/lib/carbon/whisper/'
+RRD_DIR = '/var/lib/carbon/rrd'
+DATA_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables
+LOG_DIR = '/var/log/graphite-web/'
+INDEX_FILE = '/var/lib/graphite-web/index'  # Search index file
+
+#USE_REMOTE_USER_AUTHENTICATION = True
+#LOGIN_URL = '/account/login'
+
+DATABASES = {
+    'default': {
+        'NAME': '{{ sqlite3_path }}',
+        'ENGINE': 'django.db.backends.sqlite3',
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': ''
+    }
+}
diff --git a/provisioning/salt/roots/salt/monitoring/server/config/riemann.config.jinja b/provisioning/salt/roots/salt/monitoring/server/config/riemann.config.jinja
@@ -0,0 +1,38 @@
+; -*- mode: clojure; -*-
+; vim: filetype=clojure
+
+(logging/init :file "/var/log/riemann/riemann.log")
+
+; Listen on the local interface over TCP (5555), UDP (5555), and websockets
+; (5556)
+(let [host "{{ host }}"]
+  (tcp-server :host host)
+  (udp-server :host host)
+  (ws-server  :host host))
+
+(graphite-server :host "{{ graphite_host }}" :port {{ graphite_port }})
+
+(def graph (graphite {:host "{{ graphite_downstream_host }}" :port {{ graphite_downstream_port }}}))
+
+; Expire old events from the index every 5 seconds.
+(periodically-expire 5)
+
+; Keep events in the index for 5 minutes by default.
+(let [index (default :ttl 300 (update-index (index)))]
+
+  ; Inbound events will be passed to these streams:
+  (streams
+
+    ; Index all events immediately.
+    index
+    ; Forward to graphite immediately.
+    graph
+
+    ; Calculate an overall rate of events.
+    (with {:metric 1 :host nil :state "ok" :service "events/sec"}
+      (rate 5 index))
+
+    ; Log expired events.
+    (expired
+      (fn [event] (info "expired" event)))
+))