Celery integration

task to lookup hostname
darkst4r9 · Nov 30, 2016 · 1dc639f · 1dc639f
1 parent bf53df8
commit 1dc639f
Showing 6 changed files with 175 additions and 5 deletions.
diff --git a/fir/__init__.py b/fir/__init__.py
@@ -0,0 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+
+from .celery import app as celery_app
+
+__all__ = ['celery_app']
diff --git a/fir/celery.py b/fir/celery.py
@@ -5,10 +5,14 @@
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fir.settings')
 
-app = Celery('celery',
+app = Celery('fir',
         broker='redis://localhost:6379/0',
-        backend='',
-        include=[])
+        backend='redis://localhost:6379/0'
+        )
+
+# load task module from all registered plugin
+app.autodiscover_tasks(['fir_artifacts'])
+
 
 if __name__ == '__main__':
     app.start()
diff --git a/fir/network_whois.py b/fir/network_whois.py
@@ -0,0 +1,67 @@
+from pprint import pformat
+from ipwhois import IPWhois
+
+
+class NetworkWhois:
+
+
+    @staticmethod
+    def analyze(ip):
+        links = set()
+
+        r = IPWhois(ip)
+        result = r.lookup()
+
+        print pformt(result)
+        """results.update(raw=pformat(result))"""
+
+        # Let's focus on the most specific information
+        # Which should be in the smallest subnet
+        n = 0
+        smallest_subnet = None
+
+        for network in result['nets']:
+            cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
+            if cidr_bits > n:
+                n = cidr_bits
+                smallest_subnet = network
+
+        if smallest_subnet:
+            # Create the company
+
+            print smallest_subnet['description']
+            """company = Company.get_or_create(name=smallest_subnet['description'].split("\n")[0])"""
+            """links.update(ip.active_link_to(company, 'hosting', 'Network Whois'))"""
+
+            # Link it to every email address referenced
+            print smallest_subnet['emails']
+            if smallest_subnet['emails']:
+                for email_address in smallest_subnet['emails'].split("\n"):
+                    print email_address
+                    """email = Email.get_or_create(value=email_address)"""
+                    """links.update(company.link_to(email, None, 'Network Whois'))"""
+
+            # Copy the subnet info into the main dict
+            for key in smallest_subnet:
+                print smallest_subnet
+                if smallest_subnet[key]:
+                    result["net_{}".format(key)] = smallest_subnet[key]
+
+        """
+        # Add the network whois to the context if not already present
+        for context in ip.context:
+            if context['source'] == 'network_whois':
+                break
+        else:
+            # Remove the nets info (the main one was copied)
+            result.pop("nets", None)
+            result.pop("raw", None)
+            result.pop("raw_referral", None)
+            result.pop("referral", None)
+            result.pop("query", None)
+
+            result['source'] = 'network_whois'
+            ip.add_context(result)
+        """
+        import ipdb; ipdb.set_trace()
+        return list(links)
diff --git a/fir/whois.py b/fir/whois.py
@@ -0,0 +1,70 @@
+from __future__ import absolute_import, unicode_literals
+
+from pythonwhois.net import get_whois_raw
+from pythonwhois.parse import parse_raw_whois
+from tldextract import extract
+
+"""
+def link_from_contact_info(hostname, contact, field, klass, description):
+    if contact is not None and field in contact:
+        node = klass.get_or_create(value=contact[field])
+
+        return hostname.active_link_to(node, description, 'Whois')
+    else:
+        return ()
+"""
+
+class Whois:
+
+
+    @staticmethod
+    def analyze(hostname):
+        links = set()
+
+        parts = extract(hostname)
+
+        if parts.subdomain == '':
+            should_add_context = False
+            """
+            for context in hostname.context:
+                if context['source'] == 'whois':
+                    break
+            else:
+                should_add_context = True
+                context = {'source': 'whois'}
+            """
+            data = get_whois_raw(hostname)
+            print data[0]
+            """results.update(raw=data[0])"""
+            parsed = parse_raw_whois(data, normalized=True)
+            """context['raw'] = data[0]"""
+            print data[0]
+
+            if 'creation_date' in parsed:
+                print parsed['creation_date'][0]
+
+                """context['creation_date'] = parsed['creation_date'][0]"""
+
+            if 'registrant' in parsed['contacts']:
+                print parsed['contacts']['registrant']
+
+            """
+                fields_to_extract = [
+                    ('email', Email, 'Registrant Email'),
+                    ('name', Text, 'Registrant Name'),
+                    ('organization', Text, 'Registrant Organization'),
+                    ('phone', Text, 'Registrant Phone Number'),
+                ]
+            """
+            """
+                for field, klass, description in fields_to_extract:
+                    links.update(link_from_contact_info(hostname, parsed['contacts']['registrant'], field, klass, description))
+                """
+            """
+            if should_add_context:
+                hostname.add_context(context)
+            else:
+                hostname.save()
+            """
+        import ipdb; ipdb.set_trace()
+        return list(links)
diff --git a/fir_artifacts/tasks.py b/fir_artifacts/tasks.py
@@ -1,7 +1,7 @@
 from __future__ import absolute_import, unicode_literals
-from . import app
+from celery import shared_task
 
 
-@app.task
+@shared_task
 def add(x, y):
     return x + y
diff --git a/requirements.txt b/requirements.txt
@@ -1,22 +1,46 @@
 amqp==2.1.1
+appnope==0.1.0
+backports.shutil-get-terminal-size==1.0.0
 billiard==3.5.0.2
 celery==4.0.0
 cssselect==0.9.1
+decorator==4.0.10
 dj-database-url==0.4.1
 Django==1.9.9
 django-filter==0.14.0
 djangorestframework==3.4.6
+dnspython==1.15.0
+enum34==1.1.6
 flup==1.0.2
 gunicorn==19.6.0
+idna==2.1
+ipaddr==2.1.11
+ipdb==0.10.1
+ipwhois==0.14.0
+ipython==5.1.0
+ipython-genutils==0.1.0
 kombu==4.0.0
 lxml==3.4.2
 Markdown==2.6.6
+pathlib2==2.1.0
+pexpect==4.2.1
+pickleshare==0.7.4
+prompt-toolkit==1.0.9
 psycopg2==2.6.2
+ptyprocess==0.5.1
+Pygments==2.1.3
 pymongo==2.8
 pyquery==1.2.9
 python-dateutil==2.4.1
+pythonwhois==2.4.3
 pytz==2016.7
 redis==2.10.5
+requests==2.12.2
+requests-file==1.4.1
+simplegeneric==0.8.1
 six==1.9.0
+tldextract==2.0.2
+traitlets==4.3.1
 vine==1.1.3
+wcwidth==0.1.7
 whitenoise==3.2.1