Merging PR Netflix#122

ottogroup · May 5, 2015 · 6e917f0 · 6e917f0
1 parent e1e2728
commit 6e917f0
Show file tree

Hide file tree

Showing 8 changed files with 291 additions and 11 deletions.
diff --git a/docs/jirasync.rst b/docs/jirasync.rst
@@ -0,0 +1,91 @@
+====================
+JIRA Synchronization
+====================
+
+Overview
+=============
+
+JIRA synchronization is a feature that allows Security Monkey to automatically create and update JIRA tickets based on issues it finds.
+Each ticket corresponds to a single type of issue for a single account. The tickets contain the number of open issues and a link back
+to the Security Monkey details page for that issue type.
+
+Configuring JIRA Synchronization
+===================================
+
+To use JIRA sync, you will need to create a YAML configuration file, specifying several settings.
+
+.. code-block:: yaml
+    server: https://jira.example.com
+    account: securitymonkey-service
+    password: hunter2
+    project: SECURITYMONKEY
+    issue_type: Task
+    url: https://securitymonkey.example.com
+
+``server`` - The location of the JIRA server.
+``account`` - The account with which Security Monkey will create tickets
+``password`` - The password to the account.
+``project`` - The project key where tickets will be created.
+``issue_type`` - The type of issue each ticket will be created as.
+``url`` - The URL for Security Monkey. This will be used to create links back to Security Monkey.
+``disable_transitions`` - If true, Security Monkey will not close or reopen tickets. This is false by default.
+
+Using JIRA Synchronization
+---------------------------
+
+To use JIRA sync, set the environment variable ``SECURITY_MONKEY_JIRA_SYNC`` to the location of the YAML configuration file.
+This file will be loaded once when the application starts. If set, JIRA sync will run for each account after the auditors run.
+You can also manually run a sync through ``manage.py``.
+
+``python manage.py sync_jira``
+
+Details
+=======
+
+Tickets are created with the summary:
+
+``<Issue text> - <Technology> - <Account name>``
+
+And the description:
+
+```
+This ticket was automatically created by Security Monkey. DO NOT EDIT ANYTHING BELOW THIS LINE
+Number of issues: X
+Account: Y
+View on Security Monkey
+Last Updated: TIMESTAMP
+```
+
+Security Monkey will update tickets based on the summary. If it is changed in any way, Security Monkey will
+open a new ticket instead of updating the existing one. When updating, the number of issues and last updated fields will change. Security Monkey
+will preserve all text in the description before "This ticket was automatically created by Security Monkey", and remove anything after.
+
+Security Monkey will automatically close tickets when they have zero open issues, by setting the state of the ticket to "Closed". Likewise, it will
+reopen a closed ticket if there are new open issues. This feature can be disabled by setting ``disable_transitions: true`` in the config.
+
+Justifying an issue will cause it to no longer be counted as an open issue.
+
+If an auditor is disabled, its issues will no longer be updated, opened or closed.
+
+Logs
+====
+
+JIRA sync will generate the following log lines.
+
+``Created issue <summary>`` (debug) - A new JIRA ticket was opened.
+
+``Updated issue <summary>`` (debug) - An existing ticket was updated.
+
+``Error creating ticket: <error>`` (error) - An error was encounted when creating a ticket. This could be due to a misconfigured project name, issue type
+or connectivity problems.
+
+``JIRA sync configuration missing required field: <error>`` (error) - One of the 6 required fields in the YAML configuration is missing.
+
+``Error opening JIRA sync configuration file: <error>`` (error) - Security Monkey could not open the file located at ``SECURITY_MONKEY_JIRA_SYNC``.
+
+``JIRA sync configuration file contains malformed YAML: <error>`` (error) - The YAML could not be parsed.
+
+``Syncing issues with Jira`` (info) - Auditors have finished running and JIRA sync is starting.
+
+``Error opening/closing ticket: <error>``: (error) - Security Monkey tried to set an issue to "Closed" or "Open". This error may mean that these transitions
+are named differently in your JIRA project. To disable ticket transitions, set ``disable_transitions: true`` in the config file.
diff --git a/manage.py b/manage.py
@@ -71,6 +71,14 @@ def start_scheduler():
     scheduler.setup_scheduler()
     scheduler.scheduler.start()
 
+@manager.command
+def sync_jira():
+    from security_monkey import jirasync
+    if jirasync:
+        app.logger.info('Syncing issues with Jira')
+        jirasync.sync_issues()
+    else:
+        app.logger.info('Jira sync not configured. Is SECURITY_MONKEY_JIRA_SYNC set?')
 
 @manager.option('-u', '--number', dest='number', type=unicode, required=True)
 @manager.option('-a', '--active', dest='active', type=bool, default=True)

diff --git a/requirements.txt b/requirements.txt
@@ -29,10 +29,13 @@ pika==0.9.13
 psycopg2==2.5.2
 python-dateutil==2.2
 python-memcached==1.53
+PyYAML==3.11
 requests==2.2.1
 py-bcrypt==0.4
 Sphinx==1.2.2
 gunicorn==18.0
 M2Crypto==0.22.3
 boto3==0.0.11
 dpath==1.3.2
+pyyaml==3.11
+jira==0.32
diff --git a/security_monkey/__init__.py b/security_monkey/__init__.py
@@ -67,7 +67,6 @@ def csrf_error(reason):
 
 from security_monkey.datastore import User, Role
 
-
 @login_manager.user_loader
 def load_user(email):
     """
@@ -170,3 +169,16 @@ def send_email(msg):
 from security_monkey.views.auditor_settings import AuditorSettingsPut
 api.add_resource(AuditorSettingsGet, '/api/1/auditorsettings')
 api.add_resource(AuditorSettingsPut, '/api/1/auditorsettings/<int:as_id>')
+
+## Jira Sync
+import os
+from security_monkey.jirasync import JiraSync
+jirasync_file = os.environ.get('SECURITY_MONKEY_JIRA_SYNC')
+if jirasync_file:
+    try:
+        jirasync = JiraSync(jirasync_file)
+    except Exception as e:
+        app.logger.error(repr(e))
+        jirasync = None
+else:
+    jirasync = None
diff --git a/security_monkey/auditor.py b/security_monkey/auditor.py
@@ -266,4 +266,4 @@ def _set_auditor_setting_for_issue(self, issue):
             self.index,
             issue.item.account.name))
 
-        return auditor_setting
+        return auditor_setting
diff --git a/security_monkey/jirasync.py b/security_monkey/jirasync.py
@@ -0,0 +1,141 @@
+"""
+.. module: security_monkey.jirasync
+    :platform: Unix
+    :synopsis: Creates and updates JIRA tickets based on current issues
+
+.. version:: $$VERSION$$
+.. moduleauthor:: Quentin Long <qlo@yelp.com>
+
+"""
+import datetime
+import urllib
+import yaml
+import time
+
+from jira.client import JIRA
+
+from security_monkey.datastore import Account, Technology, AuditorSettings
+from security_monkey import app
+
+
+class JiraSync(object):
+    """ Syncs auditor issues with JIRA tickets. """
+    def __init__(self, jira_file):
+        try:
+            with open(jira_file) as jf:
+                data = jf.read()
+                data = yaml.load(data)
+                self.account = data['account']
+                self.password = data['password']
+                self.project = data['project']
+                self.server = data['server']
+                self.issue_type = data['issue_type']
+                self.url = data['url']
+                self.disable_transitions = data.get('disable_transitions', False)
+        except KeyError as e:
+            raise Exception('JIRA sync configuration missing required field: {}'.format(e))
+        except IOError as e:
+            raise Exception('Error opening JIRA sync configuration file: {}'.format(e))
+        except yaml.scanner.ScannerError as e:
+            raise Exception('JIRA sync configuration file contains malformed YAML: {}'.format(e))
+
+        try:
+            self.client = JIRA(self.server, basic_auth=(self.account, self.password))
+        except Exception as e:
+            raise Exception("Error connecting to JIRA: {}".format(str(e)[:1024]))
+
+    def close_issue(self, issue):
+        try:
+            self.transition_issue(issue, 'Closed')
+        except Exception as e:
+            app.logger.error('Error closing issue {} ({}): {}'.format(issue.fields.summary, issue.key, e))
+
+    def open_issue(self, issue):
+        try:
+            self.transition_issue(issue, 'Open')
+        except Exception as e:
+            app.logger.error('Error opening issue {} ({}): {}'.format(issue.fields.summary, issue.key, e))
+
+    def transition_issue(self, issue, transition_name):
+        transitions = self.client.transitions(issue)
+        for transition in transitions:
+            if transition['name'].lower() == transition_name.lower():
+                break
+        else:
+            app.logger.error('No transition {} for issue {}'.format(transition_name, issue.key))
+            return
+        self.client.transition_issue(issue, transition['id'])
+
+    def add_or_update_issue(self, issue, technology, account, count):
+        """ Searches for existing tickets based on the summary. If one exists,
+        it will update the count and preserve any leading description text. If not, it will create a ticket. """
+        summary = '{0} - {1} - {2}'.format(issue, technology, account)
+        # Having dashes in JQL cuases it to return no results
+        summary_search = summary.replace('- ', '')
+        jql = 'project={0} and summary~"{1}"'.format(self.project, summary_search)
+        issues = self.client.search_issues(jql)
+
+        url = "{0}/#/issues/-/{1}/{2}/-/True/{3}/1/25".format(self.url, technology, account, urllib.quote(issue, ''))
+        timezone = time.tzname[time.localtime().tm_isdst]
+        description = ("This ticket was automatically created by Security Monkey. DO NOT EDIT SUMMARY OR BELOW THIS LINE\n"
+                      "Number of issues: {0}\n"
+                      "Account: {1}\n"
+                      "[View on Security Monkey|{2}]\n"
+                      "Last updated: {3} {4}".format(count, account, url, datetime.datetime.now().isoformat(), timezone))
+
+        for issue in issues:
+            # Make sure we found the exact ticket
+            if issue.fields.summary == summary:
+                old_desc = issue.fields.description
+                old_desc = old_desc[:old_desc.find('This ticket was automatically created by Security Monkey')]
+                issue.update(description=old_desc + description)
+                app.logger.debug("Updated issue {} ({})".format(summary, issue.key))
+
+                if self.disable_transitions:
+                    return
+
+                if issue.fields.status.name == 'Closed' and count:
+                    self.open_issue(issue)
+                    app.logger.debug("Reopened issue {} ({})".format(summary, issue.key))
+                elif issue.fields.status.name != 'Closed' and count == 0:
+                    self.close_issue(issue)
+                    app.logger.debug("Closed issue {} ({})".format(summary, issue.key))
+                return
+
+        # Don't open a ticket with no issues
+        if count == 0:
+            return
+
+        jira_args = {'project': {'key': self.project},
+                     'issuetype': {'name': self.issue_type},
+                     'summary': summary,
+                     'description': description}
+
+        try:
+            issue = self.client.create_issue(**jira_args)
+            app.logger.debug("Created issue {} ({})".format(summary, issue.key))
+        except Exception as e:
+            app.logger.error("Error creating issue {}: {}".format(summary, e))
+
+    def sync_issues(self, accounts=None, tech_name=None):
+        """ Runs add_or_update_issue for every AuditorSetting, filtered by technology
+        and accounts, if provided. """
+        query = AuditorSettings.query.join(
+            (Technology, Technology.id == AuditorSettings.tech_id)
+        ).join(
+            (Account, Account.id == AuditorSettings.account_id)
+        ).filter(
+            (AuditorSettings.disabled == False)
+        )
+
+        if accounts:
+            query = query.filter(Account.name.in_(accounts))
+        if tech_name:
+            query = query.filter(Technology.name == tech_name)
+
+        for auditorsetting in query.all():
+            unjustified = [issue for issue in auditorsetting.issues if not issue.justified]
+            self.add_or_update_issue(auditorsetting.issue_text,
+                                     auditorsetting.technology.name,
+                                     auditorsetting.account.name,
+                                     len(unjustified))
diff --git a/security_monkey/scheduler.py b/security_monkey/scheduler.py
@@ -15,7 +15,7 @@
 from security_monkey.monitors import all_monitors, get_monitor
 from security_monkey.reporter import Reporter
 
-from security_monkey import app, db, handler
+from security_monkey import app, db, handler, jirasync
 
 import traceback
 import time
@@ -78,17 +78,40 @@ def _find_changes(accounts, monitor, debug=True):
     cw.save()
     db.session.close()
 
-def _audit_changes(accounts, auditors, send_report, debug=True):
-    """ Runs auditors on all items """
-    for au in auditors:
-        au.audit_all_objects()
-        if send_report:
-            report = au.create_report()
-            au.email_report(report)
-        au.save_issues()
 
+def _audit_changes(accounts, monitor, send_report, debug=True):
+    """ Runs an auditors on all items and, if enabled, syncs Jira """
+    accounts = __prep_accounts__(accounts)
+    au = monitor.auditor_class(accounts=accounts, debug=True)
+    au.audit_all_objects()
+
+    if send_report:
+        report = au.create_report()
+        au.email_report(report)
+
+    au.save_issues()
     db.session.close()
 
+    if jirasync:
+        app.logger.info('Syncing {} issues on {} with Jira'.format(monitor.index, accounts))
+        jirasync.sync_issues(accounts, monitor.index)
+
+def run_account(account):
+    """
+    This should be refactored into Reporter.
+    Runs the watchers/auditors for each account.
+    Does not run the alerter.
+    Times the operations and logs those results.
+    """
+    app.logger.info("Starting work on account {}.".format(account))
+    time1 = time.time()
+    for monitor in all_monitors():
+        find_changes(account, monitor)
+        app.logger.info("Account {} is done with {}".format(account, monitor.index))
+    time2 = time.time()
+    app.logger.info('Run Account %s took %0.1f s' % (account, (time2-time1)))
+
+
 pool = ThreadPool(core_threads=25, max_threads=30, keepalive=0)
 scheduler = Scheduler(standalone=True, threadpool=pool, coalesce=True, misfire_grace_time=30)
 

diff --git a/setup.py b/setup.py
@@ -60,5 +60,7 @@
             'M2Crypto==0.22.3',
             'boto3==0.0.11',
             'dpath==1.3.2'
+            'pyyaml==3.11',
+            'jira==0.32'
     ]
 )