[py_common/proxy] backend-based workers

Co-authored-by: Copilot <copilot@github.com>

Author: feederbox826

scrapers/py_common/proxy.py

@@ -21,14 +21,94 @@
 import requests
 import cloudscraper
 
-def _is_blocked(res):
-  if res.status_code not in (403, 503):
-    return False
-  body = res.text[:4096].lower()
-  return any(marker in body for marker in (
-    "cloudflare", "cf-ray", "cf-chl", "challenge-platform",
-    "just a moment", "attention required",
-  ))
+class RequestBackend:
+  name = "backend"
+
+  @staticmethod
+  def is_blocked(res):
+    if res.status_code not in (403, 503):
+      return False
+    body = res.text[:4096].lower()
+    return any(marker in body for marker in (
+      "cloudflare", "cf-ray", "cf-chl", "challenge-platform",
+      "just a moment", "attention required",
+    ))
+
+  def request(self, method, url, **kwargs):
+    raise NotImplementedError
+
+class RequestsBackend(RequestBackend):
+  name = "requests"
+
+  def __init__(self, proxies=None, useragent=None):
+    self.session = requests.Session()
+    if proxies:
+      self.session.proxies = proxies
+    if useragent == "inherit":
+      ua = get_useragent()
+      self.session.headers.update({"User-Agent": ua})
+    elif useragent:
+      self.session.headers.update({"User-Agent": useragent})
+
+  def _apply_cache(self, url):
+    cache_entry = cookie_cache.get(url)
+    if cache_entry:
+      log.debug(f"[proxy] Using cache for {url}")
+      for cookie in cache_entry['cookies']:
+        self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path'])
+      if cache_entry['useragent']:
+        self.session.headers.update({"User-Agent": cache_entry['useragent']})
+    return bool(cache_entry)
+
+  def request(self, method, url, **kwargs):
+    self._apply_cache(url)
+    try:
+      res = self.session.request(method, url, **kwargs)
+      if not self.is_blocked(res):
+        return res
+      log.warning(f"[proxy] requests blocked ({res.status_code}).")
+    except requests.exceptions.RequestException as e:
+      log.warning(f"[proxy] Request failed: {e}.")
+    raise Exception("Requests backend failed")
+
+class CloudscraperBackend(RequestBackend):
+  name = "cloudscraper"
+
+  def __init__(self, proxies=None):
+    self.scraper = cloudscraper.create_scraper()
+    if proxies:
+      self.scraper.proxies = proxies
+
+  def request(self, method, url, **kwargs):
+    try:
+      res = self.scraper.request(method, url, **kwargs)
+      if not self.is_blocked(res):
+        return res
+      log.warning(f"[proxy] Cloudscraper blocked ({res.status_code}).")
+    except requests.exceptions.RequestException as e:
+      log.warning(f"[proxy] Cloudscraper request failed: {e}")
+    raise Exception("Cloudscraper backend failed")
+
+class FlareSolverrBackend(RequestBackend):
+  name = "flaresolverr"
+
+  def request(self, method, url, **kwargs):
+    if check_flaresolverr(FLARESOLVERR_URL):
+      log.info(f"[proxy] trying FlareSolverr for {url}")
+      # HEAD is not supported
+      if method == "head":
+        method = "get"
+        log.warning("[proxy] HEAD not supported by FlareSolverr, using GET instead")
+      try:
+        post_data = (kwargs.get("json") or kwargs.get("data")) if method == "post" else None
+        return flaresolverr_req(url, method=method, postData=post_data, proxy=PROXY_URL)
+      except Exception as e:
+        log.warning(f"[proxy] FlareSolverr request failed: {e}")
+    else:
+      raise Exception("FlareSolverr not detected")
+    raise Exception("FlareSolverr backend failed")
+
+## END REWRITE
 
 FLARESOLVERR_URL = os.environ.get("FLARESOLVERR_URL", "http://localhost:8191/v1")
 
@@ -48,7 +128,7 @@ def check_flaresolverr(url):
 
 @cache_to_disk(ttl=86400)
 def get_useragent() -> str:
-  chrome_ua = requests.get("https://jnrbsn.github.io/user-agents/user-agents.json").json()[3]
+  chrome_ua = requests.get("https://feederbox826.github.io/user-agents/user-agents.json").json()[3]
   return chrome_ua
 
 def flaresolverr_req(url, method="get", postData=None, proxy=None) -> requests.Response:
@@ -107,75 +187,53 @@ def update(self):
 
 cookie_cache = CookieCache()
 
+class BackendManager:
+  def __init__(self, backends=None):
+    proxies = { "http": PROXY_URL, "https": PROXY_URL } if PROXY_URL else {}
+    if not backends:
+      self.backends = [
+        RequestsBackend(proxies=proxies, useragent="inherit"),
+        CloudscraperBackend(proxies=proxies),
+      ]
+      if check_flaresolverr(FLARESOLVERR_URL):
+        self.backends.append(FlareSolverrBackend())
+    else:
+      # populate backends based on name
+      backend_classes = {
+        "requests": RequestsBackend,
+        "cloudscraper": CloudscraperBackend,
+        "flaresolverr": FlareSolverrBackend,
+      }
+      self.backends = []
+      for name in backends:
+        cls = backend_classes.get(name)
+        if cls:
+          self.backends.append(cls(proxies=proxies))
+        else:
+          log.warning(f"[proxy] Unknown backend specified: {name}")
+
+  def request(self, method, url, **kwargs):
+    for backend in self.backends:
+      try:
+        return backend.request(method, url, **kwargs)
+      except Exception as e:
+        log.debug(f"[proxy] {backend.name} failed: {e}")
+    raise Exception("All backends failed")
+
 class StashRequests:
-  def __init__(self, cloudflare=False, useragent="inherit"):
-    self.session = requests.Session()
+  def __init__ (self, cloudflare=False, useragent="inherit"):
     self.proxies = { "http": PROXY_URL, "https": PROXY_URL } if PROXY_URL else {}
-    self.session.proxies = self.proxies
-    if useragent == "inherit":
-      ua = get_useragent()
-      self.session.headers.update({"User-Agent": ua})
-    elif useragent:
-      self.session.headers.update({"User-Agent": useragent})
     self.cloudflare = cloudflare
-    self._cloudscraper = None
-
-  def _apply_cache(self, url):
-    cache_entry = cookie_cache.get(url)
-    if cache_entry:
-      log.debug(f"[proxy] Using cache for {url}")
-      for cookie in cache_entry['cookies']:
-        self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie['domain'], path=cookie['path'])
-      if cache_entry['useragent']:
-        self.session.headers.update({"User-Agent": cache_entry['useragent']})
-    return bool(cache_entry)
-
-  def get_cloudscraper(self):
-    if not self._cloudscraper:
-      self._cloudscraper = cloudscraper.create_scraper()
-      self._cloudscraper.proxies = self.proxies
-    return self._cloudscraper
-
-  def _request(self, method, url, **kwargs):
-    has_cache = self._apply_cache(url)
-    if has_cache or not self.cloudflare:
-      try:
-        res = self.session.request(method, url, **kwargs)
-        if not _is_blocked(res):
-          return res
-        log.warning(f"[proxy] requests blocked ({res.status_code}). Trying cloudscraper.")
-        self.cloudflare = True
-      except requests.exceptions.RequestException as e:
-        log.warning(f"[proxy] Request failed: {e}. Trying cloudscraper.")
-        self.cloudflare = True
-    try:
-      scraper = self.get_cloudscraper()
-      res = scraper.request(method, url, **kwargs)
-      if not _is_blocked(res):
-        return res
-      log.warning(f"[proxy] Cloudscraper blocked ({res.status_code}).")
-    except requests.exceptions.RequestException as e:
-      log.warning(f"[proxy] Cloudscraper request failed: {e}")
-    if check_flaresolverr(FLARESOLVERR_URL):
-      log.info(f"[proxy] trying FlareSolverr for {url}")
-      # HEAD is not supported
-      if method == "head":
-        method = "get"
-        log.warning("[proxy] HEAD not supported by FlareSolverr, using GET instead")
-      try:
-        post_data = (kwargs.get("json") or kwargs.get("data")) if method == "post" else None
-        return flaresolverr_req(url, method=method, postData=post_data, proxy=PROXY_URL)
-      except Exception as e:
-        log.warning(f"[proxy] FlareSolverr request failed: {e}")
-    raise Exception("All request methods failed")
+    self.useragent = useragent
+    self.manager = BackendManager()
 
   def get(self, url, **kwargs):
-    return self._request("get", url, **kwargs)
+    return self.manager.request("get", url, **kwargs)
 
   def post(self, url, **kwargs):
-    return self._request("post", url, **kwargs)
-  
+    return self.manager.request("post", url, **kwargs)
+
   def head(self, url, **kwargs):
-    return self._request("head", url, **kwargs)
+    return self.manager.request("head", url, **kwargs)
 
 stash_requests = StashRequests()