Correctly get page content after JS Execution

- Thanks for @AbdullahY36 for the heads-up.

**Update reference** https://developer.mozilla.org/en-US/docs/Web/API/Document/DOMContentLoaded_event

scrapling/engines/camo.py

@@ -87,7 +87,8 @@ class CamoufoxEngine:
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -105,7 +106,7 @@ class CamoufoxEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,

scrapling/engines/pw.py

@@ -199,7 +199,8 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -217,7 +218,7 @@ class PlaywrightEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,

scrapling/engines/toolbelt/custom.py

@@ -28,10 +28,14 @@ class Response:
     def adaptor(self) -> Union[Adaptor, None]:
         """Generate Adaptor instance from this response if possible, otherwise return None"""
         automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
-        if self.content:
+        if self.text:
-            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+            # For playwright that will be the response after all JS executed
-        elif self.text:
             return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.content:
+            # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
+            # To get response Bytes after the load states
+            # Reference: https://playwright.dev/python/docs/api/class-page
+            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
         return None
 
     def __repr__(self):