v0.3.12

Author: D4Vinci

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
     - name: Install all browsers dependencies
       run: |
         python3 -m pip install --upgrade pip
-        python3 -m pip install playwright>=1.55.0 patchright>=1.55.0 camoufox
+        python3 -m pip install playwright==1.56.0 patchright==1.56.0 camoufox>=0.4.11
 
     - name: Get Playwright version
       id: playwright-version
@@ -83,7 +83,11 @@ jobs:
     - name: Install Playwright browsers
       run: |
         echo "Cache hit: ${{ steps.playwright-cache.outputs.cache-hit }}"
-        python3 -m playwright install chromium
+        if [ "${{ steps.playwright-cache.outputs.cache-hit }}" != "true" ]; then
+          python3 -m playwright install chromium
+        else
+          echo "Skipping install - using cached Playwright browsers"
+        fi
         python3 -m playwright install-deps chromium firefox
 
     - name: Get Camoufox version
@@ -108,7 +112,11 @@ jobs:
     - name: Install Camoufox browser
       run: |
         echo "Cache hit: ${{ steps.camoufox-cache.outputs.cache-hit }}"
-        python3 -m camoufox fetch --browserforge
+        if [ "${{ steps.camoufox-cache.outputs.cache-hit }}" != "true" ]; then
+          python3 -m camoufox fetch --browserforge
+        else
+          echo "Skipping fetch - using cached Camoufox browser"
+        fi
 
     # Cache tox environments
     - name: Cache tox environments

docs/fetching/dynamic.md

@@ -93,6 +93,7 @@ Scrapling provides many options with this fetcher and its session classes. To ma
 |       stealth       | Enables stealth mode; you should always check the documentation to see what the stealth mode does currently.                                                                                                                                                                                                                                                                                                                |    ✔️    |
 |     real_chrome     | If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch and use an instance of your browser.                                                                                                                                                                                                                                                                                        |    ✔️    |
 |       locale        | Set the locale for the browser if wanted. The default value is `en-US`.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
+|     timezone_id     | Set the timezone for the browser if wanted.                                                                                                                                                                                                                                                                                                                                                                                 |    ✔️    |
 |       cdp_url       | Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |    user_data_dir    | Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. **Only Works with sessions**                                                                                                                                                                                                                                               |    ✔️    |
 |     extra_flags     | A list of additional browser flags to pass to the browser on launch.                                                                                                                                                                                                                                                                                                                                                        |    ✔️    |

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "scrapling"
 # Static version instead of a dynamic version so we can get better layer caching while building docker, check the docker file to understand
-version = "0.3.11"
+version = "0.3.12"
 description = "Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!"
 readme = {file = "docs/README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}
@@ -59,22 +59,22 @@ classifiers = [
 dependencies = [
     "lxml>=6.0.2",
     "cssselect>=1.3.0",
-    "orjson>=3.11.4",
+    "orjson>=3.11.5",
     "tldextract>=5.3.0",
 ]
 
 [project.optional-dependencies]
 fetchers = [
     "click>=8.3.0",
-    "curl_cffi>=0.13.0",
-    "playwright>=1.56.0",
-    "patchright>=1.56.0",
+    "curl_cffi>=0.14.0",
+    "playwright==1.56.0",
+    "patchright==1.56.0",
     "camoufox>=0.4.11",
     "geoip2>=5.2.0",
     "msgspec>=0.20.0",
 ]
 ai = [
-    "mcp>=1.23.0",
+    "mcp>=1.24.0",
     "markdownify>=1.2.0",
     "scrapling[fetchers]",
 ]

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3.11"
+__version__ = "0.3.12"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 from typing import Any, TYPE_CHECKING

scrapling/cli.py

@@ -2,8 +2,6 @@
 from subprocess import check_output
 from sys import executable as python_executable
 
-from curl_cffi.requests import impersonate
-
 from scrapling.core.utils import log
 from scrapling.engines.toolbelt.custom import Response
 from scrapling.core.utils._shell import _CookieParser, _ParseHeaders

scrapling/engines/_browsers/_base.py

@@ -40,7 +40,7 @@ def __init__(self, max_pages: int = 1):
         self.context: BrowserContext | Any = None
         self._closed = False
 
-    def __create__(self):
+    def start(self):
         pass
 
     def close(self):  # pragma: no cover
@@ -59,7 +59,7 @@ def close(self):  # pragma: no cover
         self._closed = True
 
     def __enter__(self):
-        self.__create__()
+        self.start()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -145,7 +145,7 @@ def __init__(self, max_pages: int = 1):
         self._closed = False
         self._lock = Lock()
 
-    async def __create__(self):
+    async def start(self):
         pass
 
     async def close(self):
@@ -164,7 +164,7 @@ async def close(self):
         self._closed = True
 
     async def __aenter__(self):
-        await self.__create__()
+        await self.start()
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
@@ -281,6 +281,7 @@ def __validate__(self, **params):
         self._wait_selector_state = config.wait_selector_state
         self._extra_flags = config.extra_flags
         self._selector_config = config.selector_config
+        self._timezone_id = config.timezone_id
         self._additional_args = config.additional_args
         self._page_action = config.page_action
         self._user_data_dir = config.user_data_dir
@@ -304,6 +305,7 @@ def __initiate_browser_options__(self):
                     self._stealth,
                     self._hide_canvas,
                     self._disable_webgl,
+                    self._timezone_id,
                     tuple(self._extra_flags) if self._extra_flags else tuple(),
                 )
             )

scrapling/engines/_browsers/_camoufox.py

@@ -102,16 +102,19 @@ def __init__(self, **kwargs: Unpack[CamoufoxSession]):
         self.__validate__(**kwargs)
         super().__init__(max_pages=self._max_pages)
 
-    def __create__(self):
+    def start(self):
         """Create a browser for this instance and context."""
-        self.playwright = sync_playwright().start()
-        self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
+        if not self.playwright:
+            self.playwright = sync_playwright().start()
+            self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)
 
-        if self._init_script:  # pragma: no cover
-            self.context.add_init_script(path=self._init_script)
+            if self._init_script:  # pragma: no cover
+                self.context.add_init_script(path=self._init_script)
 
-        if self._cookies:  # pragma: no cover
-            self.context.add_cookies(self._cookies)
+            if self._cookies:  # pragma: no cover
+                self.context.add_cookies(self._cookies)
+        else:
+            raise RuntimeError("Session has been already started")
 
     def _cloudflare_solver(self, page: Page) -> None:  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed
@@ -299,18 +302,21 @@ def __init__(self, **kwargs: Unpack[CamoufoxSession]):
         self.__validate__(**kwargs)
         super().__init__(max_pages=self._max_pages)
 
-    async def __create__(self):
+    async def start(self):
         """Create a browser for this instance and context."""
-        self.playwright: AsyncPlaywright = await async_playwright().start()
-        self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
-            **self.launch_options
-        )
+        if not self.playwright:
+            self.playwright: AsyncPlaywright = await async_playwright().start()
+            self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
+                **self.launch_options
+            )
 
-        if self._init_script:  # pragma: no cover
-            await self.context.add_init_script(path=self._init_script)
+            if self._init_script:  # pragma: no cover
+                await self.context.add_init_script(path=self._init_script)
 
-        if self._cookies:
-            await self.context.add_cookies(self._cookies)  # pyright: ignore [reportArgumentType]
+            if self._cookies:
+                await self.context.add_cookies(self._cookies)  # pyright: ignore [reportArgumentType]
+        else:
+            raise RuntimeError("Session has been already started")
 
     async def _cloudflare_solver(self, page: async_Page):  # pragma: no cover
         """Solve the cloudflare challenge displayed on the playwright page passed. The async version

scrapling/engines/_browsers/_config_tools.py

@@ -70,6 +70,7 @@ def _launch_kwargs(
     stealth,
     hide_canvas,
     disable_webgl,
+    timezone_id,
     extra_flags: Tuple,
 ) -> Tuple:
     """Creates the arguments we will use while launching playwright's browser"""
@@ -79,6 +80,7 @@ def _launch_kwargs(
 
     launch_kwargs = {
         "locale": locale,
+        "timezone_id": timezone_id or None,
         "headless": headless,
         "args": base_args,
         "color_scheme": "dark",  # Bypasses the 'prefersLightColor' check in creepjs

scrapling/engines/_browsers/_controllers.py

@@ -77,6 +77,7 @@ def __init__(self, **kwargs: Unpack[PlaywrightSession]):
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+        :param timezone_id: Set the timezone for the browser if wanted.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
@@ -95,24 +96,27 @@ def __init__(self, **kwargs: Unpack[PlaywrightSession]):
         self.__validate__(**kwargs)
         super().__init__(max_pages=self._max_pages)
 
-    def __create__(self):
+    def start(self):
         """Create a browser for this instance and context."""
-        sync_context = sync_patchright if self._stealth else sync_playwright
+        if not self.playwright:
+            sync_context = sync_patchright if self._stealth else sync_playwright
 
-        self.playwright: Playwright = sync_context().start()  # pyright: ignore [reportAttributeAccessIssue]
+            self.playwright: Playwright = sync_context().start()  # pyright: ignore [reportAttributeAccessIssue]
 
-        if self._cdp_url:  # pragma: no cover
-            self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
-                **self.context_options
-            )
-        else:
-            self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
+            if self._cdp_url:  # pragma: no cover
+                self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(
+                    **self.context_options
+                )
+            else:
+                self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)
 
-        if self._init_script:  # pragma: no cover
-            self.context.add_init_script(path=self._init_script)
+            if self._init_script:  # pragma: no cover
+                self.context.add_init_script(path=self._init_script)
 
-        if self._cookies:  # pragma: no cover
-            self.context.add_cookies(self._cookies)
+            if self._cookies:  # pragma: no cover
+                self.context.add_cookies(self._cookies)
+        else:
+            raise RuntimeError("Session has been already started")
 
     def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
         """Opens up the browser and do your request based on your chosen options.
@@ -209,6 +213,7 @@ def __init__(self, **kwargs: Unpack[PlaywrightSession]):
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
         :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
+        :param timezone_id: Set the timezone for the browser if wanted.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
@@ -227,25 +232,28 @@ def __init__(self, **kwargs: Unpack[PlaywrightSession]):
         self.__validate__(**kwargs)
         super().__init__(max_pages=self._max_pages)
 
-    async def __create__(self):
+    async def start(self):
         """Create a browser for this instance and context."""
-        async_context = async_patchright if self._stealth else async_playwright
+        if not self.playwright:
+            async_context = async_patchright if self._stealth else async_playwright
 
-        self.playwright: AsyncPlaywright = await async_context().start()  # pyright: ignore [reportAttributeAccessIssue]
+            self.playwright: AsyncPlaywright = await async_context().start()  # pyright: ignore [reportAttributeAccessIssue]
 
-        if self._cdp_url:
-            browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
-            self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
-        else:
-            self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
-                **self.launch_options
-            )
+            if self._cdp_url:
+                browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)
+                self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
+            else:
+                self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
+                    **self.launch_options
+                )
 
-        if self._init_script:  # pragma: no cover
-            await self.context.add_init_script(path=self._init_script)
+            if self._init_script:  # pragma: no cover
+                await self.context.add_init_script(path=self._init_script)
 
-        if self._cookies:
-            await self.context.add_cookies(self._cookies)  # pyright: ignore
+            if self._cookies:
+                await self.context.add_cookies(self._cookies)  # pyright: ignore
+        else:
+            raise RuntimeError("Session has been already started")
 
     async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
         """Opens up the browser and do your request based on your chosen options.

scrapling/engines/_browsers/_validators.py

@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
     load_dom: bool = True
     wait_selector_state: SelectorWaitStates = "attached"
     user_data_dir: str = ""
+    timezone_id: str = ""
     extra_flags: Optional[List[str]] = None
     selector_config: Optional[Dict] = {}
     additional_args: Optional[Dict] = {}