From 239e2c73bfe458f42b11f35cfb9ed9fe1d3a4479 Mon Sep 17 00:00:00 2001
From: Karim shoair <D4Vinci@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:11:46 +0200
Subject: [PATCH] Correctly get page content after JS Execution

- Thanks for @AbdullahY36 for the heads-up.

**Update reference** https://developer.mozilla.org/en-US/docs/Web/API/Document/DOMContentLoaded_event
---
 scrapling/engines/camo.py            |  5 +++--
 scrapling/engines/pw.py              |  5 +++--
 scrapling/engines/toolbelt/custom.py | 10 +++++++---
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 61731fc..6763b2b 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -87,7 +87,8 @@ class CamoufoxEngine:
             if self.extra_headers:
                 page.set_extra_http_headers(self.extra_headers)
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -105,7 +106,7 @@ class CamoufoxEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 72f571a..e3aa654 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -199,7 +199,8 @@ class PlaywrightEngine:
                 page.add_init_script(path=js_bypass_path('screen_props.js'))
                 page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
 
-            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
@@ -217,7 +218,7 @@ class PlaywrightEngine:
 
             response = Response(
                 url=res.url,
-                text=res.text(),
+                text=page.content(),
                 content=res.body(),
                 status=res.status,
                 reason=res.status_text,
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 9a1ca33..3688be2 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -28,10 +28,14 @@ class Response:
     def adaptor(self) -> Union[Adaptor, None]:
         """Generate Adaptor instance from this response if possible, otherwise return None"""
         automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
-        if self.content:
-            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
-        elif self.text:
+        if self.text:
+            # For playwright that will be the response after all JS executed
             return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+        elif self.content:
+            # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
+            # To get response Bytes after the load states
+            # Reference: https://playwright.dev/python/docs/api/class-page
+            return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
         return None
 
     def __repr__(self):