[ADD] Profiler

stanislav-web · Jan 12, 2017 · f1e1cf8 · f1e1cf8
1 parent 72105c0
commit f1e1cf8
Show file tree

Hide file tree

Showing 14 changed files with 222 additions and 26,446 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ opendoor.egg-info/
 /dist/
 /syslog/exceptions.log
 /tmp/
+/pycallgraph.png
diff --git a/TODO.txt b/TODO.txt
diff --git a/data/cookies.txt b/data/cookies.txt
diff --git a/data/directories.dat b/data/directories.dat
@@ -4237,26 +4237,24 @@ aerobics/
 aes/
 aestatement/
 af/
-afadmin
+afadmin/
 afb/
 afbeeldingen/
 afc/
 afegir/
 aff/
-affadmin
+affadmin/
 affichage/
 affiche/
 affil/
-affiliate
 affiliate.php
 affiliate/
-affiliate_admin
+affiliate_admin/
 affiliate_info/
 affiliate_terms.php
 affiliate_terms/
 affiliateimages/
 affiliatelogin/
-affiliates
 affiliates.sql
 affiliates/
 affiliatewiz/
@@ -10579,7 +10577,7 @@ exiar/
 exif/
 exit/
 exp/
-expadmin
+expadmin/
 expedia/
 expediade/
 expediauk/
@@ -10601,7 +10599,6 @@ explore/
 explorer/
 expo/
 expop/
-export
 export/
 export_db/
 export_files/
@@ -10699,7 +10696,7 @@ faculty/
 faculty_staff/
 facultystaff/
 fad/
-fadmin
+fadmin/
 fail/
 failed/
 failure/
@@ -22395,7 +22392,7 @@ stadtplan/
 staff
 staff/
 staff_directory/
-staffadmin
+staffadmin/
 staffonly/
 staffs/
 stage/

diff --git a/keyboard.py b/keyboard.py
@@ -0,0 +1,40 @@
+import termios, fcntl, sys, os
+
+def get_char_keyboard():
+  fd = sys.stdin.fileno()
+
+  oldterm = termios.tcgetattr(fd)
+  newattr = termios.tcgetattr(fd)
+  newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
+  termios.tcsetattr(fd, termios.TCSANOW, newattr)
+
+  c = None
+  try:
+    c = sys.stdin.read(1)
+  except IOError: pass
+
+  termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
+
+  return c
+
+def get_char_keyboard_nonblock():
+  fd = sys.stdin.fileno()
+
+  oldterm = termios.tcgetattr(fd)
+  newattr = termios.tcgetattr(fd)
+  newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
+  termios.tcsetattr(fd, termios.TCSANOW, newattr)
+
+  oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
+  fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)
+
+  c = None
+
+  try:
+    c = sys.stdin.read(1)
+  except IOError: pass
+
+  termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
+  fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)
+
+  return c
diff --git a/opendoor.py b/opendoor.py
@@ -19,6 +19,9 @@
 
 import sys
 
+from pycallgraph import PyCallGraph
+from pycallgraph.output import GraphvizOutput
+
 try:
     import urllib3
 
@@ -31,9 +34,10 @@
     from src import Controller , SrcError
 
     try:
-
-        bootstrap = Controller()
-        bootstrap.run()
+        # using profiler
+        with PyCallGraph(output=GraphvizOutput()):
+            bootstrap = Controller()
+            bootstrap.run()
     except SrcError as e:
         sys.exit(e.message)
 
diff --git a/setup.cfg b/setup.cfg
@@ -14,6 +14,7 @@ excludes = data/exclusions.dat
 proxies = data/proxies.dat
 subdomains = data/subdomains.dat
 useragents = data/useragents.dat
+cookies = data/cookies.dat
 tmplist    = tmp/list.tmp
 
 [system]

diff --git a/src/core/keyboard/__init__.py b/src/core/keyboard/__init__.py
diff --git a/src/core/keyboard/keyboard.py b/src/core/keyboard/keyboard.py
diff --git a/src/core/options/config.py b/src/core/options/config.py
@@ -81,6 +81,15 @@ class Config:
             "help": "Request timeout",
             "type": int
         },
+        {
+            "group": "request",
+            "args": None,
+            "argl": "--cookies",
+            "default": "",
+            "action": "store",
+            "help": "Request cookies from cookies.txt",
+            "type": str
+        },
         {
             "group": "debug",
             "args": None,

diff --git a/src/lib/browser/browser.py b/src/lib/browser/browser.py
@@ -82,22 +82,33 @@ def scan(self):
 
         self._get_lines(self._scan,
             params={'host' : self._host, 'port' : self._port, 'scheme' : self._scheme},
-            callback= getattr(self, '_process'.format())
+            callback= getattr(self, '_create_pool'.format())
         )
 
         pass
 
-
-    def _process(self):
+    def _create_pool(self):
         """
-        Process adding items into queue pool
+        Create items pool
 
         :return: None
         """
 
         if self._total_lines() == self.count_in_queue():
             tpl.info(key='scanning', host=self._host)
-            # self.join_to_queue()
+            self.read_from_queue(self.__load)
         else:
             self._debug_progress(self.count_in_queue(), self._total_lines())
-            pass
+            pass
+
+    def __load(self, threadno, url):
+        """
+        Process adding items into queue pool
+
+        :return: None
+        """
+        if False == self.is_pooled(threadno):
+            tpl.line_log("{0} - {1}".format(threadno, url))
+        else:
+            tpl.message("\n")
+            tpl.info("{0} - {1}".format(threadno, url))
diff --git a/src/lib/browser/pool.py b/src/lib/browser/pool.py
@@ -22,6 +22,8 @@ class Pool:
     """Pool class"""
 
     def __init__(self, threads):
+
+        self.pool = {}
         self.queue = Queue()
         self.threads = threads
 
@@ -31,9 +33,6 @@ def get_pool_instance(self):
     def add_to_queue(self, item):
         self.queue.put(item)
 
-    def queue_join(self):
-        self.queue.join()
-
     def count_in_queue(self):
         return self.queue.qsize()
 
@@ -44,14 +43,23 @@ def read_from_queue(self, process):
             worker = Thread(target=self.process_queue, args=(i, self.queue))
             worker.setDaemon(True)
             worker.start()
-        self.queue_join()
+        self.queue.join()
+
+    def is_pooled(self, i):
+
+        if i in self.pool:
+            return True
+        else:
+            return False
 
     def process_queue(self, i, q):
 
         while not q.empty():  # check that the queue isn't empty
-            print '%s: Looking for the next enclosure' % i
+            # print '%s: Looking for the next enclosure' % i
             item = q.get()  # get the item from the queue
-            print '%s: Downloading:' % i, item
-            self.callback(item)
+            if False == self.is_pooled(i):
+                self.pool[i] = True
+            # print '%s: Downloading:' % i, item
+            self.callback(i, item)
             q.task_done()
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ opendoor.egg-info/ @@
     /dist/
     /syslog/exceptions.log
     /tmp/
+    /pycallgraph.png