fix(proxy-rotation): removed max_shape duplicate

ScrapeGraphAI · May 13, 2024 · 5d6d996 · 5d6d996
1 parent 1e9a564
commit 5d6d996
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 15 deletions.
diff --git a/examples/openai/proxy.py b/examples/openai/proxy.py
diff --git a/examples/openai/smart_scraper_openai.py b/examples/openai/smart_scraper_openai.py
@@ -28,10 +28,10 @@
             "server": "broker",
             "criteria": {
                 "anonymous": True,
-                # "secure": True,
+                "secure": True,
                 "countryset": {"IT"},
-                "timeout": 5.0,
-                "max_shape": 2
+                "timeout": 10.0,
+                "max_shape": 3
             },
         },
     }

diff --git a/scrapegraphai/utils/proxy_rotation.py b/scrapegraphai/utils/proxy_rotation.py
@@ -161,7 +161,13 @@ def _search_proxy(proxy: Proxy) -> ProxySettings:
     Returns:
         A 'playwright' compliant proxy configuration.
     """
-    server = search_proxy_servers(**proxy.get("criteria", {}))[0]
+
+
+    # remove max_shape from criteria 
+    criteria = proxy.get("criteria", {}).copy()
+    criteria.pop("max_shape", None)
+
+    server = search_proxy_servers(max_shape=1, **criteria)[0]
 
     return {"server": server}