version 3.0

SecurityFTW · Aug 8, 2018 · 94a8a0d · 94a8a0d
1 parent 52e32e4
commit 94a8a0d
Show file tree

Hide file tree

Showing 453 changed files with 6,901 additions and 3,517 deletions.
diff --git a/AWS_Audit_Report.png b/AWS_Audit_Report.png
diff --git a/Dockerfile b/Dockerfile
diff --git a/GCP_Audit_Report.png b/GCP_Audit_Report.png
diff --git a/LICENSE b/LICENSE
diff --git a/README.md b/README.md
diff --git a/_config.yml b/_config.yml
diff --git a/cs-suite.png b/cs-suite.png
diff --git a/cs.py b/cs.py
diff --git a/modules/__init__.py b/modules/__init__.py
diff --git a/modules/awsaudit.py b/modules/awsaudit.py
diff --git a/modules/azureaudit.py b/modules/azureaudit.py
diff --git a/modules/gcpaudit.py b/modules/gcpaudit.py
@@ -0,0 +1,20 @@
+from __future__ import print_function
+import subprocess
+import os
+import time
+import webbrowser
+
+timestmp = time.strftime("%Y%m%d-%H%M%S")
+
+def gcp_audit(project_name):
+    """ This function just calls the G-Scout to Audit GCP """
+    subprocess.call(['mkdir', '-p', 'reports/GCP/%s/%s' % (project_name, timestmp)])
+    print ("Starting GCP Audit")
+    subprocess.call(['python', 'gscout.py', 'project', project_name], cwd='tools/G-Scout')
+    if os.path.exists("tools/G-Scout/Report Output/%s" % (project_name)):
+        subprocess.check_output(['mv tools/G-Scout/Report\ Output/%s/* reports/GCP/%s/%s/' % (project_name, project_name, timestmp)], shell=True)
+        subprocess.check_output(['rm -rf tools/G-Scout/Report\ Output/%s' % (project_name)], shell=True)
+        webbrowser.open('file://' + os.path.realpath("./reports/GCP/%s/%s/All Ports Open to All.html") % (project_name, timestmp))
+        fin = os.path.realpath("./reports/GCP/%s/%s/All\ Ports\ Open\ to\ All.html") % (project_name, timestmp)
+        print ("THE FINAL REPORT IS LOCATED AT -------->  %s" % (fin))
+
diff --git a/modules/localaudit.py b/modules/localaudit.py
@@ -0,0 +1,77 @@
+from __future__ import print_function
+import subprocess
+import json
+from IPy import IP
+
+def get_account_alias():
+    account_details = subprocess.check_output(['aws iam list-account-aliases'], shell=True)
+    account_details = json.loads(str(account_details))
+    try:
+        return account_details['AccountAliases'][0]
+    except IndexError:
+        return None
+
+def get_account_id():
+    caller_identity = subprocess.check_output(['aws sts get-caller-identity'], shell=True)
+    caller_identity = json.loads(str(caller_identity))
+    try:
+        return caller_identity['Account']
+    except IndexError:
+        return None
+
+account_name = get_account_alias() or get_account_id()
+
+def local_audit(audit_ip, user_name, pem_file, password):
+    win_path = 'tools/Windows-Workstation-and-Server-Audit'
+    lyn_path = 'tools/lynis'
+    ip = IP(audit_ip)
+    type = ip.iptype()
+    default_region = subprocess.check_output(['aws', 'configure', 'get', 'region']).strip()
+    if type == 'PUBLIC':
+        operating_sys = subprocess.check_output(['aws', 'ec2', 'describe-instances', '--region', '%s' % default_region, '--filters','Name=ip-address,Values=%s' % (audit_ip), '--query', 'Reservations[*].Instances[*].[Platform]', '--output', 'text']).strip()
+        private_ip = subprocess.check_output(['aws', 'ec2', 'describe-instances', '--region', '%s' % default_region, '--filters', 'Name=ip-address,Values=%s' % (audit_ip), '--query', 'Reservations[*].Instances[*].[PrivateIpAddress]', '--output', 'text']).strip()
+        public_ip = audit_ip
+    elif type == 'PRIVATE':
+        operating_sys = subprocess.check_output(['aws', 'ec2', 'describe-instances', '--region', '%s' % default_region, '--filters', 'Name=network-interface.addresses.private-ip-address,Values=%s' % (audit_ip), '--query', 'Reservations[*].Instances[*].[Platform]', '--output', 'text']).strip()
+        public_ip = subprocess.check_output(['aws', 'ec2', 'describe-instances', '--region', '%s' % default_region, '--filters', 'Name=network-interface.addresses.private-ip-address,Values=%s' % (audit_ip), '--query', 'Reservations[*].Instances[*].[PublicIpAddress]', '--output', 'text']).strip()
+        private_ip = audit_ip
+    if public_ip == 'None':
+        public_ip = ""
+    else:
+        dns_name = subprocess.check_output(['host', public_ip]).strip().split(' ')[4]
+    if operating_sys == 'windows':
+        print ("WINDOWS BOX FOUND!!!")
+        if (audit_ip and not (user_name or pem_file or password)):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip], cwd=win_path)
+        elif audit_ip and user_name and not (pem_file or password):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, user_name], cwd=win_path)
+        elif audit_ip and pem_file and not (user_name or password):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, "", pem_file], cwd=win_path)
+        elif audit_ip and password and not (user_name or pem_file):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, "", "", password], cwd=win_path)
+        elif audit_ip and user_name and password and not (pem_file):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, "", password], cwd=win_path)
+        elif audit_ip and user_name and pem_file and not (password):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, pem_file], cwd=win_path)
+        elif audit_ip and password and pem_file and not (password):
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, "", pem_file, password], cwd=win_path)
+        else:
+            subprocess.call(['./windows_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, pem_file, password], cwd=win_path)
+    else:
+        print ("LINUX BOX FOUND!!!")
+        if (audit_ip and not (user_name or pem_file or password)):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip], cwd=lyn_path)
+        elif audit_ip and user_name and not (pem_file or password):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, user_name], cwd=lyn_path)
+        elif audit_ip and pem_file and not (user_name or password):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, "", pem_file], cwd=lyn_path)
+        elif audit_ip and password and not (user_name or pem_file):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, "", "", password], cwd=lyn_path)
+        elif audit_ip and user_name and password and not (pem_file):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, "", password], cwd=lyn_path)
+        elif audit_ip and user_name and pem_file and not (password):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, pem_file, ""], cwd=lyn_path)
+        elif audit_ip and password and pem_file and not (password):
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, "", pem_file, password], cwd=lyn_path)
+        else:
+            subprocess.call(['./lynis_remote.sh', account_name, dns_name, private_ip, public_ip, user_name, pem_file, password], cwd=lyn_path)
diff --git a/modules/merger.py b/modules/merger.py
@@ -0,0 +1,229 @@
+import glob
+import json
+import os
+import webbrowser
+import subprocess
+import awsaudit
+
+account_name = awsaudit.account_name
+timestmp = awsaudit.timestmp
+script_json = awsaudit.script_json
+
+def trusted_advisor_to_json():
+    data = []
+    with open('reports/AWS/aws_audit/%s/%s/final_report/trusted.json' % (account_name, timestmp), 'r') as f:
+        for line in f:
+            k = json.loads(line)
+            data.append(k['check'])
+    data = set(data)
+
+    for i in data:
+        with open('reports/AWS/aws_audit/%s/%s/final_report/%s.txt' % (account_name, timestmp, i), 'w+') as f:
+            with open('reports/AWS/aws_audit/%s/%s/final_report/trusted.json' % (account_name, timestmp), 'r') as j:
+                for line in j:
+                    k = json.loads(line)
+                    if k['check'] == i:
+                        f.write(line)
+    final_json = {}
+    report = []
+    for f in glob.glob("reports/AWS/aws_audit/%s/%s/final_report/*.txt" % (account_name, timestmp)):
+        dict = {}
+        data = []
+        with open(f, 'r') as g:
+            for line in g:
+                new_dict = {}
+                j = json.loads(line)
+                dict['check'] = j['check']
+                new_dict['check_no'] = j['check_no']
+                new_dict['score'] = j['score']
+                new_dict['level'] = j['level']
+                new_dict['type'] = j['type']
+                new_dict['region'] = j['region']
+                new_dict['value'] = j['value']
+                data.append(new_dict)
+        dict['data'] = data
+        report.append(dict)
+        final_json['report'] = report
+    with open('reports/AWS/aws_audit/%s/%s/final_report/final_json' % (account_name, timestmp), 'w') as f:
+        f.write(json.dumps(final_json))
+    for f in glob.glob("reports/AWS/aws_audit/%s/%s/final_report/*.txt" % (account_name, timestmp)):
+        os.remove(f)
+    json_to_html_trusted()
+
+def json_to_html_trusted():
+    with open('./reports/AWS/aws_audit/%s/%s/final_report/trusted_advisor.html' % (account_name, timestmp), 'w') as f:
+        with open('./tools/prowler/template1.txt', 'r') as g:
+             for line in g:
+                  f.write(line)
+        with open('./reports/AWS/aws_audit/%s/%s/final_report/final_json' % (account_name, timestmp), 'r') as json_data:
+             final = json.load(json_data)
+             for i in final['report']:
+                  f.write('<div class="col-xs-6 col-sm-3 col-md-3 item">\n')
+                  f.write('<div class="thumbnail">\n')
+                  f.write('<div class="caption">\n')
+                  flag = 0
+                  for g in i['data']:
+                     if g['type'] in ['warning', 'error']:
+                         flag = 1
+                  if flag == 0:
+                     f.write('<div class="grid" style="background-color: green;">')
+                  else:
+                     f.write('<div class="grid" style="background-color: red;">')
+                  f.write('<h5>%s</h5>\n' %(i['check']))
+                  f.write('</div>')
+                  for k in i['data']:
+                       if k['type'] in ['warning', 'error']:
+                           f.write('<p><span style="color:red">Warning: </span>%s</p>\n' %(k['value']))
+                       else:
+                           f.write('<p>%s</p>\n' %(k['value']))
+                  f.write('</div>\n')
+                  f.write('</div>\n')
+                  f.write('</div>\n')
+        with open('./tools/prowler/template2.txt', 'r') as k:
+             for line in k:
+                 f.write(line)
+
+
+def json_to_final_json():
+    report = []
+    for f in glob.glob("reports/AWS/aws_audit/%s/%s/delta/*.json" %(account_name, timestmp)):
+        dict = {}
+        data = []
+        with open(f, 'r') as g:
+             for line in g:
+                 new_dict = {}
+                 j = json.loads(line)
+                 dict['check'] = j['check']
+                 new_dict['check_no'] = j['check_no']
+                 new_dict['score'] = j['score']
+                 new_dict['level'] = j['level']
+                 new_dict['type'] = j['type']
+                 new_dict['region'] = j['region']
+                 new_dict['value'] = j['value']
+                 data.append(new_dict)
+        dict['data'] = data
+        report.append(dict)
+        script_json['report'] = report
+    with open('reports/AWS/aws_audit/%s/%s/delta/final_json' % (account_name, timestmp), 'w') as f:
+         f.write(json.dumps(script_json))
+    for i in script_json['report']:
+        if i['check'] in ['CDN_AUDIT', 'CERT_AUDIT', 'DNS_AUDIT', 'ELB_AUDIT']:
+            with open('reports/AWS/aws_audit/%s/%s/delta/webnet.json' % (account_name, timestmp), 'a+') as f:
+                f.write(json.dumps(i))
+                f.write('\n')
+        elif i['check'] in ['ELASTIC_CACHE_AUDIT', 'ELASTIC_SEARCH_AUDIT', 'RDS_AUDIT', 'REDSHIFT_AUDIT']:
+            with open('reports/AWS/aws_audit/%s/%s/delta/datastores.json' % (account_name, timestmp), 'a+') as f:
+                f.write(json.dumps(i))
+                f.write('\n')
+        elif i['check'] in ['CLOUD_FORMATION_AUDIT', 'SES_AUDIT', 'SNS_AUDIT']:
+            with open('reports/AWS/aws_audit/%s/%s/delta/notification.json' % (account_name, timestmp), 'a+') as f:
+                f.write(json.dumps(i))
+                f.write('\n')
+        else:
+            with open('reports/AWS/aws_audit/%s/%s/delta/configs.json' % (account_name, timestmp), 'a+') as f:
+                f.write(json.dumps(i))
+                f.write('\n')
+
+def json_to_html_prowler():
+    with open('./reports/AWS/aws_audit/%s/%s/delta/prowler_report.html' % (account_name, timestmp), 'w') as f:
+        with open('./tools/prowler/template1.txt', 'r') as g:
+             for line in g:
+                  f.write(line)
+        with open('./tools/prowler/final_json', 'r') as json_data:
+             final = json.load(json_data)
+             for i in final['report']:
+                  f.write('<div class="col-xs-6 col-sm-3 col-md-3 item">\n')
+                  f.write('<div class="thumbnail">\n')
+		  f.write('<div class="caption">\n')
+                  flag = 0
+                  for g in i['data']:
+                     if g['type'] == 'WARNING':
+                         flag = 1
+                  if flag == 0:
+                     f.write('<div class="grid" style="background-color: green;">')
+                  else:
+                     f.write('<div class="grid" style="background-color: red;">')
+                  f.write('<h5>%s</h5>\n' %(i['check']))
+                  f.write('</div>')
+                  for k in i['data']:
+                       if k['type'] == 'WARNING':
+                           f.write('<p><span style="color:red">Warning: </span>%s</p>\n' %(k['value']))
+                       else:
+                           f.write('<p>%s</p>\n' %(k['value']))
+                  f.write('</div>\n')
+                  f.write('</div>\n')
+                  f.write('</div>\n')
+        with open('./tools/prowler/template2.txt', 'r') as k:
+             for line in k:
+                 f.write(line)
+
+def json_to_html(file, new_file):
+    with open(new_file, 'w') as f:
+        with open('./tools/prowler/template1.txt', 'r') as g:
+            for line in g:
+                f.write(line)
+        with open(file, 'r') as json_data:
+             for line in json_data:
+                 line = str(line)
+                 final = json.loads(line)
+                 f.write('<div class="col-xs-6 col-sm-3 col-md-3 item">\n')
+                 f.write('<div class="thumbnail">\n')
+                 f.write('<div class="caption">\n')
+                 flag = 0
+                 for g in final['data']:
+                     if g['type'] == 'WARNING':
+                         flag = 1
+                 if flag == 0:
+                     f.write('<div class="grid" style="background-color: green;">')
+                 else:
+                     f.write('<div class="grid" style="background-color: red;">')
+                 f.write('<h5>%s</h5>\n' %(final['check']))
+                 f.write('</div>')
+                 for k in final['data']:
+                     if k['type'] == 'WARNING':
+                          f.write('<p><span style="color:red">Warning: </span>%s</p>\n' %(k['value']))
+                     else:
+                          f.write('<p>%s<p>\n' %(k['value']))
+                 f.write('</div>')
+                 f.write('</div>')
+                 f.write('</div>')
+        with open('./tools/prowler/template2.txt', 'r') as k:
+             for line in k:
+                 f.write(line)
+
+
+def merge_json():
+    with open('reports/AWS/aws_audit/%s/%s/delta/final_json' % (account_name, timestmp), 'r') as f:
+        for line in f:
+            j1 = json.loads(line)
+    with open('./tools/prowler/final_json', 'r') as k:
+        for line in k:
+            j2 = json.loads(line)
+
+    j1['report'].append(j2['report'])
+    with open('reports/AWS/aws_audit/%s/%s/delta/final_json' % (account_name, timestmp), 'w') as f:
+        f.write(json.dumps(j1))
+    os.remove('./tools/prowler/final_json')
+
+
+def merge():
+    if os.stat('reports/AWS/aws_audit/%s/%s/final_report/trusted.json' % (account_name, timestmp)).st_size != 0:
+        trusted_advisor_to_json()
+    json_to_final_json()
+    json_to_html_prowler()
+    json_to_html('./reports/AWS/aws_audit/%s/%s/delta/webnet.json' % (account_name, timestmp),
+                 './reports/AWS/aws_audit/%s/%s/delta/webnet.html' % (account_name, timestmp))
+    json_to_html('./reports/AWS/aws_audit/%s/%s/delta/datastores.json' % (account_name, timestmp),
+                 './reports/AWS/aws_audit/%s/%s/delta/datastores.html' % (account_name, timestmp))
+    json_to_html('./reports/AWS/aws_audit/%s/%s/delta/notification.json' % (account_name, timestmp),
+                 './reports/AWS/aws_audit/%s/%s/delta/notification.html' % (account_name, timestmp))
+    json_to_html('./reports/AWS/aws_audit/%s/%s/delta/configs.json' % (account_name, timestmp),
+                 './reports/AWS/aws_audit/%s/%s/delta/configs.html' % (account_name, timestmp))
+    merge_json()
+    subprocess.check_output(
+        ['cp -R ./tools/template/* ./reports/AWS/aws_audit/%s/%s/final_report/' % (account_name, timestmp)], shell=True)
+    subprocess.check_output(['rm ./reports/AWS/aws_audit/%s/%s/final_report/report_azure.html' % (account_name, timestmp)], shell=True)
+    webbrowser.open('file://' + os.path.realpath("./reports/AWS/aws_audit/%s/%s/final_report/report.html")
+                    % (account_name, timestmp))
+    fin = os.path.realpath("./reports/AWS/aws_audit/%s/%s/final_report/report.html") % (account_name, timestmp)
+    print ("THE FINAL REPORT IS LOCATED AT -------->  %s" % (fin))
diff --git a/requirements-virtual.txt b/requirements-virtual.txt
diff --git a/requirements.txt b/requirements.txt
diff --git a/scripts/audit_aws_certs.py b/scripts/audit_aws_certs.py
diff --git a/setup.py b/setup.py
diff --git a/tools/G-Scout/LICENSE b/tools/G-Scout/LICENSE
diff --git a/tools/G-Scout/add_finding.py b/tools/G-Scout/add_finding.py
diff --git a/tools/G-Scout/category.py b/tools/G-Scout/category.py
diff --git a/tools/Scout2/.coveragerc b/tools/Scout2/.coveragerc
diff --git a/tools/Scout2/.gitignore b/tools/Scout2/.gitignore
@@ -10,6 +10,8 @@ venv/
 # Packages
 *.egg
 *.egg-info
+.eggs
+.cache/
 dist
 build
 eggs
@@ -41,8 +43,18 @@ nosetests.xml
 
 # In case people do a `git add .` ...
 *.csv
+tests/data/ruleset-test.json
 
 # Data folder and files
 scout2-report*
 inc-awsconfig*
 report-*
+
+# PyCharm
+.idea/
+
+# Mac cruft
+.DS_Store
+
+# Emacs backups
+*~
diff --git a/tools/Scout2/.travis.yml b/tools/Scout2/.travis.yml
@@ -20,11 +20,11 @@ script:
 #  - '[ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] && nosetests tests/test-utils_sns.py || false'
     - nosetests --with-coverage tests/test-rules-ruleset.py
     - nosetests --with-coverage tests/test-rules-processingengine.py
-    - nosetests --with-coverage tests/test-scout2.py
+    - nosetests --with-coverage --nocapture tests/test-scout2.py
 
 # use container-base infrastructure
 sudo: false
 
-# Update test coverage
-after_success:
-    - coveralls
+# Update test coverage -- only when run locally
+#after_success:
+#    - coveralls
diff --git a/tools/Scout2/AWSScout2/__init__.py b/tools/Scout2/AWSScout2/__init__.py
diff --git a/tools/Scout2/AWSScout2/__listall__.py b/tools/Scout2/AWSScout2/__listall__.py
@@ -107,7 +107,7 @@ def main():
                 rule.keys = ['name']
 
         # Prepare the output format
-        (lines, template) = format_listall_output(args.format_file, 'foo', args.format, rule)
+        (lines, template) = format_listall_output(args.format_file[0], None, args.format, rule)
 
         # Print the output
         printInfo(generate_listall_output(lines, resources, aws_config, template, []))
diff --git a/tools/Scout2/AWSScout2/__main__.py b/tools/Scout2/AWSScout2/__main__.py
@@ -138,3 +138,4 @@ def main():
         url = 'file://%s' % os.path.abspath(html_report_path)
         webbrowser.open(url, new=2)
 
+    return 0
diff --git a/tools/Scout2/AWSScout2/cli_parser.py b/tools/Scout2/AWSScout2/cli_parser.py
@@ -66,7 +66,7 @@ def __init__(self, default_args = None):
                                  dest='rules_dir',
                                  default=[],
                                  nargs='+',
-                                 help='Path to custom rule files.')
+                                 help='Path to directories where custom rules are defined.')
         self.parser.add_argument('--generator-dir',
                                  dest='generator_dir',
                                  default=DEFAULT_REPORT_DIR,

diff --git a/tools/Scout2/AWSScout2/configs/__init__.py b/tools/Scout2/AWSScout2/configs/__init__.py
Original file line number	Diff line number	Diff line change
Expand Up		@@ -138,3 +138,4 @@ def main():
		url = 'file://%s' % os.path.abspath(html_report_path)
		webbrowser.open(url, new=2)

		return 0