diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py index 61731fc..6763b2b 100644 --- a/scrapling/engines/camo.py +++ b/scrapling/engines/camo.py @@ -87,7 +87,8 @@ class CamoufoxEngine: if self.extra_headers: page.set_extra_http_headers(self.extra_headers) - res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded") + res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + page.wait_for_load_state(state="domcontentloaded") if self.network_idle: page.wait_for_load_state('networkidle') @@ -105,7 +106,7 @@ class CamoufoxEngine: response = Response( url=res.url, - text=res.text(), + text=page.content(), content=res.body(), status=res.status, reason=res.status_text, diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py index 72f571a..e3aa654 100644 --- a/scrapling/engines/pw.py +++ b/scrapling/engines/pw.py @@ -199,7 +199,8 @@ class PlaywrightEngine: page.add_init_script(path=js_bypass_path('screen_props.js')) page.add_init_script(path=js_bypass_path('playwright_fingerprint.js')) - res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded") + res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None) + page.wait_for_load_state(state="domcontentloaded") if self.network_idle: page.wait_for_load_state('networkidle') @@ -217,7 +218,7 @@ class PlaywrightEngine: response = Response( url=res.url, - text=res.text(), + text=page.content(), content=res.body(), status=res.status, reason=res.status_text, diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py index 9a1ca33..3688be2 100644 --- a/scrapling/engines/toolbelt/custom.py +++ b/scrapling/engines/toolbelt/custom.py @@ -28,10 +28,14 @@ class Response: def adaptor(self) -> Union[Adaptor, None]: """Generate Adaptor instance from this response if possible, otherwise return None""" automatch_domain = self.adaptor_arguments.pop('automatch_domain', None) - if self.content: - return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments) - elif self.text: + if self.text: + # For playwright that will be the response after all JS executed return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments) + elif self.content: + # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content() + # To get response Bytes after the load states + # Reference: https://playwright.dev/python/docs/api/class-page + return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments) return None def __repr__(self):