Skip to content

Commit 82b0962

Browse files
D4VinciD4Vinci
committed
perf: Improving all browser-based fetches by ~15%
1 parent 970ad40 commit 82b0962

4 files changed

Lines changed: 194 additions & 137 deletions

File tree

‎scrapling/engines/_browsers/_base.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,6 @@ def _get_page(
6060

6161
return self.page_pool.add_page(page)
6262

63-
@staticmethod
64-
def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:
65-
"""Get value with request-level priority over session-level"""
66-
return request_value if request_value is not sentinel_value else session_value
67-
6863
def get_pool_stats(self) -> Dict[str, int]:
6964
"""Get statistics about the current page pool"""
7065
return {

‎scrapling/engines/_browsers/_camoufox.py‎

Lines changed: 33 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
)
1717
from playwright._impl._errors import Error as PlaywrightError
1818

19-
from ._validators import validate, CamoufoxConfig
19+
from ._validators import validate_fetch as _validate
2020
from ._base import SyncSession, AsyncSession, StealthySessionMixin
2121
from scrapling.core.utils import log
2222
from scrapling.core._types import (
@@ -297,23 +297,22 @@ def fetch(
297297
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
298298
:return: A `Response` object.
299299
"""
300-
# Validate all resolved parameters
301-
params = validate(
302-
dict(
303-
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
304-
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
305-
wait=self._get_with_precedence(wait, self.wait, _UNSET),
306-
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
307-
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
308-
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
309-
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
310-
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
311-
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
312-
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
313-
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
314-
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
315-
),
316-
CamoufoxConfig,
300+
params = _validate(
301+
[
302+
("google_search", google_search, self.google_search),
303+
("timeout", timeout, self.timeout),
304+
("wait", wait, self.wait),
305+
("page_action", page_action, self.page_action),
306+
("extra_headers", extra_headers, self.extra_headers),
307+
("disable_resources", disable_resources, self.disable_resources),
308+
("wait_selector", wait_selector, self.wait_selector),
309+
("wait_selector_state", wait_selector_state, self.wait_selector_state),
310+
("network_idle", network_idle, self.network_idle),
311+
("load_dom", load_dom, self.load_dom),
312+
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
313+
("selector_config", selector_config, self.selector_config),
314+
],
315+
_UNSET,
317316
)
318317

319318
if self._closed: # pragma: no cover
@@ -617,22 +616,22 @@ async def fetch(
617616
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
618617
:return: A `Response` object.
619618
"""
620-
params = validate(
621-
dict(
622-
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
623-
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
624-
wait=self._get_with_precedence(wait, self.wait, _UNSET),
625-
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
626-
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
627-
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
628-
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
629-
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
630-
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
631-
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
632-
solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
633-
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
634-
),
635-
CamoufoxConfig,
619+
params = _validate(
620+
[
621+
("google_search", google_search, self.google_search),
622+
("timeout", timeout, self.timeout),
623+
("wait", wait, self.wait),
624+
("page_action", page_action, self.page_action),
625+
("extra_headers", extra_headers, self.extra_headers),
626+
("disable_resources", disable_resources, self.disable_resources),
627+
("wait_selector", wait_selector, self.wait_selector),
628+
("wait_selector_state", wait_selector_state, self.wait_selector_state),
629+
("network_idle", network_idle, self.network_idle),
630+
("load_dom", load_dom, self.load_dom),
631+
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
632+
("selector_config", selector_config, self.selector_config),
633+
],
634+
_UNSET,
636635
)
637636

638637
if self._closed: # pragma: no cover

‎scrapling/engines/_browsers/_controllers.py‎

Lines changed: 31 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from scrapling.core.utils import log
1818
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
19-
from ._validators import validate, PlaywrightConfig
19+
from ._validators import validate_fetch as _validate
2020
from scrapling.core._types import (
2121
Dict,
2222
List,
@@ -224,22 +224,21 @@ def fetch(
224224
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
225225
:return: A `Response` object.
226226
"""
227-
# Validate all resolved parameters
228-
params = validate(
229-
dict(
230-
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
231-
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
232-
wait=self._get_with_precedence(wait, self.wait, _UNSET),
233-
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
234-
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
235-
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
236-
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
237-
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
238-
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
239-
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
240-
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
241-
),
242-
PlaywrightConfig,
227+
params = _validate(
228+
[
229+
("google_search", google_search, self.google_search),
230+
("timeout", timeout, self.timeout),
231+
("wait", wait, self.wait),
232+
("page_action", page_action, self.page_action),
233+
("extra_headers", extra_headers, self.extra_headers),
234+
("disable_resources", disable_resources, self.disable_resources),
235+
("wait_selector", wait_selector, self.wait_selector),
236+
("wait_selector_state", wait_selector_state, self.wait_selector_state),
237+
("network_idle", network_idle, self.network_idle),
238+
("load_dom", load_dom, self.load_dom),
239+
("selector_config", selector_config, self.selector_config),
240+
],
241+
_UNSET,
243242
)
244243

245244
if self._closed: # pragma: no cover
@@ -471,22 +470,21 @@ async def fetch(
471470
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
472471
:return: A `Response` object.
473472
"""
474-
# Validate all resolved parameters
475-
params = validate(
476-
dict(
477-
google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
478-
timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
479-
wait=self._get_with_precedence(wait, self.wait, _UNSET),
480-
page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
481-
extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
482-
disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
483-
wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
484-
wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
485-
network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
486-
load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
487-
selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
488-
),
489-
PlaywrightConfig,
473+
params = _validate(
474+
[
475+
("google_search", google_search, self.google_search),
476+
("timeout", timeout, self.timeout),
477+
("wait", wait, self.wait),
478+
("page_action", page_action, self.page_action),
479+
("extra_headers", extra_headers, self.extra_headers),
480+
("disable_resources", disable_resources, self.disable_resources),
481+
("wait_selector", wait_selector, self.wait_selector),
482+
("wait_selector_state", wait_selector_state, self.wait_selector_state),
483+
("network_idle", network_idle, self.network_idle),
484+
("load_dom", load_dom, self.load_dom),
485+
("selector_config", selector_config, self.selector_config),
486+
],
487+
_UNSET,
490488
)
491489

492490
if self._closed: # pragma: no cover

0 commit comments

Comments
 (0)