v0.3.11

Author: D4Vinci

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "scrapling"
 # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
-version = "0.3.10"
+version = "0.3.11"
 description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
 readme = {file = "docs/README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
@@ -74,7 +74,7 @@ fetchers = [
     "msgspec>=0.20.0",
 ]
 ai = [
-    "mcp>=1.19.0",
+    "mcp>=1.23.0",
     "markdownify>=1.2.0",
     "scrapling[fetchers]",
 ]

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3.10"
+__version__ = "0.3.11"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 from typing import Any, TYPE_CHECKING

scrapling/core/shell.py

@@ -314,15 +314,15 @@ def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
             return None
 
 
-def _unpack_signature(func):
+def _unpack_signature(func, signature_name=None):
     """
     Unpack TypedDict from Unpack[TypedDict] annotations in **kwargs and reconstruct the signature.
 
     This allows the interactive shell to show individual parameters instead of just **kwargs, similar to how IDEs display them.
     """
     try:
         sig = signature(func)
-        func_name = getattr(func, "__name__", None)
+        func_name = signature_name or getattr(func, "__name__", None)
 
         # Check if this function has known parameters
         if func_name not in Signatures_map:
@@ -467,7 +467,7 @@ def update_page(self, result):  # pragma: no cover
 
         return result
 
-    def create_wrapper(self, func, get_signature=True):
+    def create_wrapper(self, func, get_signature=True, signature_name=None):
         """Create a wrapper that preserves function signature but updates page"""
 
         @wraps(func)
@@ -477,7 +477,7 @@ def wrapper(*args, **kwargs):
 
         if get_signature:
             # Explicitly preserve and unpack signature for IPython introspection and autocompletion
-            wrapper.__signature__ = _unpack_signature(func)  # pyright: ignore
+            wrapper.__signature__ = _unpack_signature(func, signature_name)  # pyright: ignore
         else:
             wrapper.__signature__ = signature(func)  # pyright: ignore
 
@@ -492,7 +492,7 @@ def get_namespace(self):
         put = self.create_wrapper(self.__Fetcher.put)
         delete = self.create_wrapper(self.__Fetcher.delete)
         dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)
-        stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch)
+        stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch")
         curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False)
 
         # Create the namespace dictionary

scrapling/engines/_browsers/_base.py

@@ -105,7 +105,7 @@ def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None):
         """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
         try:
             page.wait_for_load_state("networkidle", timeout=timeout)
-        except PlaywrightError:
+        except (PlaywrightError, Exception):
             pass
 
     def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):
@@ -221,7 +221,7 @@ async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[
         """Wait for the page to become idle (no network activity) even if there are never-ending requests."""
         try:
             await page.wait_for_load_state("networkidle", timeout=timeout)
-        except PlaywrightError:
+        except (PlaywrightError, Exception):
             pass
 
     async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.3.10
+version = 0.3.11
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!

tox.ini

@@ -15,11 +15,6 @@ deps =
     camoufox
     -r{toxinidir}/tests/requirements.txt
 extras = ai,shell
-commands_pre =
-    # Install browsers in the tox virtual environment
-    python -m playwright install chromium
-    python -m playwright install-deps chromium firefox
-    python -m camoufox fetch --browserforge
 commands =
     # Run browser tests without parallelization (avoid browser conflicts)
     pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose