[fix] fix log

abop · Mar 7, 2017 · 9df3168 · 9df3168
1 parent 83ead7f
commit 9df3168
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/Manager/ProxyManager.py b/Manager/ProxyManager.py
@@ -38,7 +38,8 @@ def refresh(self):
             proxy_set = set()
             # fetch raw proxy
             for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                proxy_set.add(proxy)
+                if proxy.strip():
+                    proxy_set.add(proxy.strip())
 
             # store raw proxy
             self.db.changeTable(self.raw_proxy_queue)

diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py
@@ -54,7 +54,6 @@ def freeProxyFirst(page=10):
         # 页数不用太多， 后面的全是历史IP， 可用性不高
 
         for url in url_list:
-            print url
             tree = getHtmlTree(url)
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:

diff --git a/Schedule/ProxyRefreshSchedule.py b/Schedule/ProxyRefreshSchedule.py
@@ -44,8 +44,8 @@ def valid_proxy(self):
         """
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
+        self.log.info('%s start valid proxy' % time.ctime())
         while raw_proxy:
-            self.log.info('%s start valid proxy' % time.ctime())
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
@@ -54,9 +54,9 @@ def valid_proxy(self):
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
-                    self.log.info('proxy: %s validation passes')
+                    self.log.debug('proxy: %s validation passes' % raw_proxy)
             except Exception, e:
-                print e
+                self.log.debug('proxy: %s validation fail' % raw_proxy)
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
@@ -74,7 +74,6 @@ def main(process_num=10):
     pl = []
     for num in range(process_num):
         proc = Process(target=refresh_pool, args=())
-        # proc.daemon = True
         pl.append(proc)
 
     for num in range(process_num):