[update] unProxyCheck时,剩余代理少于POOL_SIZE_MIN时执行抓取

abop · Feb 23, 2021 · dcee407 · dcee407
1 parent 9d39874
commit dcee407
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 41 deletions.
diff --git a/api/proxyApi.py b/api/proxyApi.py
@@ -60,17 +60,6 @@ def get():
     proxy = proxy_handler.get()
     return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
-@app.route('/update_check_count', methods=['GET'])
-def update_check_count():
-    proxy_str = request.args.get('proxy')
-    proxy = proxy_handler.update_check_count(proxy_str)
-    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
-
-@app.route('/update_fail_count', methods=['GET'])
-def update_fail_count():
-    proxy_str = request.args.get('proxy')
-    proxy = proxy_handler.update_fail_count(proxy_str)
-    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 @app.route('/pop/')
 def pop():

diff --git a/db/redisClient.py b/db/redisClient.py
@@ -40,19 +40,17 @@ def __init__(self, **kwargs):
         kwargs.pop("username")
         self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str=None):
+    def get(self):
         """
         返回一个代理
         :return:
         """
         proxies = self.__conn.hkeys(self.name)
-        if not proxies:
-            return None
-        if proxy_str is None:
-            proxy = choice(proxies)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
         else:
-            proxy = proxy_str
-        return self.__conn.hget(self.name, proxy)
+            return False
 
     def put(self, proxy_obj):
         """

diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -10,6 +10,7 @@ ChangeLog
 2. 修复 **66代理** 采集; (2020-11-04)
 3. 修复 **全网代理** 采集, 解决HTML端口加密问题; (2020-11-04)
 4. 新增 **代理盒子** 免费源; (2020-11-04)
+5. 新增`POOL_SIZE_MIN`配置项, runProxyCheck时, 剩余代理少于POOL_SIZE_MIN触发抓取; (2020-11-04)
 
 .. _#493: https://github.com/jhao104/proxy_pool/issues/493
 

diff --git a/handler/proxyHandler.py b/handler/proxyHandler.py
@@ -35,24 +35,6 @@ def get(self):
             return Proxy.createFromJson(proxy)
         return None
 
-    def update_check_count(self, proxy_str):
-        proxy = self.db.get(proxy_str)
-        if not proxy:
-            return None
-        proxy = Proxy.createFromJson(proxy)
-        proxy.check_count += 1
-        self.db.update(proxy)
-        return proxy
-
-    def update_fail_count(self, proxy_str):
-        proxy = self.db.get(proxy_str)
-        if not proxy:
-            return None
-        proxy = Proxy.createFromJson(proxy)
-        proxy.fail_count += 1
-        self.db.update(proxy)
-        return proxy
-
     def pop(self):
         """
         return and delete a useful proxy

diff --git a/helper/check.py b/helper/check.py
@@ -41,8 +41,8 @@ def __proxyCheck(proxy):
         proxy_obj.check_count += 1
         proxy_obj.last_status = 1
         proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        # if proxy_obj.fail_count > 0:
-        #     proxy_obj.fail_count -= 1
+        if proxy_obj.fail_count > 0:
+            proxy_obj.fail_count -= 1
         return proxy_obj
     else:
         proxy_obj.check_count += 1
@@ -90,7 +90,7 @@ def run(self):
                     self.log.info('ProxyCheck - {}  : {} pass'.format(self.name, proxy.proxy.ljust(23)))
                     self.proxy_handler.put(proxy)
                 else:
-                    if proxy.fail_count / proxy.check_count > self.conf.maxFailRate:
+                    if proxy.fail_count > self.conf.maxFailCount:
                         self.log.info('ProxyCheck - {}  : {} fail, count {} delete'.format(self.name,
                                                                                            proxy.proxy.ljust(23),
                                                                                            proxy.fail_count))

diff --git a/helper/scheduler.py b/helper/scheduler.py
@@ -45,7 +45,7 @@ def _runProxyCheck():
         runChecker("use", proxy_queue)
 
 
-def run():
+def runScheduler():
     _runProxyFetch()
 
     timezone = ConfigHandler().timezone
@@ -70,4 +70,4 @@ def run():
 
 
 if __name__ == '__main__':
-    run()
+    runScheduler()