[update] 代理抓取时添加异常捕获

abop · Jan 18, 2018 · 275288d · 275288d
1 parent a4c341d
commit 275288d
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 10 deletions.
diff --git a/Api/ProxyApi.py b/Api/ProxyApi.py
@@ -20,13 +20,10 @@
 from flask import Flask, jsonify, request
 from Util.GetConfig import GetConfig
 
-
-
 from Manager.ProxyManager import ProxyManager
 
 app = Flask(__name__)
 
-
 api_list = {
     'get': u'get an usable proxy',
     # 'refresh': u'refresh proxy pool',
@@ -78,5 +75,6 @@ def run():
     config = GetConfig()
     app.run(host=config.host_ip, port=config.host_port)
 
+
 if __name__ == '__main__':
     run()
diff --git a/ProxyGetter/getFreeProxy.py b/ProxyGetter/getFreeProxy.py
@@ -65,7 +65,7 @@ def freeProxySecond(proxy_number=100):
         :return:
         """
         url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
-                proxy_number)
+            proxy_number)
         request = WebRequest()
         # html = request.get(url).content
         # content为未解码，text为解码后的字符串
@@ -162,6 +162,7 @@ def freeProxySeventh():
             for tr in proxy_list[1:]:
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
 
+
 if __name__ == '__main__':
     gg = GetFreeProxy()
     # for e in gg.freeProxyFirst():
@@ -171,12 +172,12 @@ def freeProxySeventh():
     #     print(e)
     #
     # for e in gg.freeProxyThird():
-        # print(e)
+    # print(e)
 
     # for e in gg.freeProxyFourth():
     #     print(e)
 
-    #for e in gg.freeProxyFifth():
+    # for e in gg.freeProxyFifth():
     #    print(e)
 
     # for e in gg.freeProxySixth():

diff --git a/Run/main.py b/Run/main.py
@@ -15,7 +15,6 @@
 import sys
 from multiprocessing import Process
 
-
 sys.path.append('../')
 
 from Api.ProxyApi import run as ProxyApiRun
@@ -38,5 +37,6 @@ def run():
     for p in p_list:
         p.join()
 
+
 if __name__ == '__main__':
-   run()
+    run()
diff --git a/Schedule/ProxyCheck.py b/Schedule/ProxyCheck.py
@@ -37,7 +37,7 @@ def run(self):
             proxy_item = self.db.pop()
             while proxy_item:
                 proxy = proxy_item.get('proxy')
-                counter = proxy_item.get('value')
+                counter = proxy_item.get('value', 1)
                 if validUsefulProxy(proxy):
                     # 验证通过计数器加1
                     if counter and int(counter) < 1:
@@ -48,7 +48,7 @@ def run(self):
                 else:
                     self.log.info('ProxyCheck: {} validation fail'.format(proxy))
                     # 验证失败，计数器减1
-                    if counter and int(counter) <= -FAIL_COUNT:
+                    if counter and int(counter) <= FAIL_COUNT:
                         self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
                         self.db.delete(proxy)
                     else: