freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make sure to default utf-8 as encoding and strip unwanted single or double quotes

Browse Source
This commit is contained in:
andyfcx
2024-11-21 14:54:38 +08:00
parent e9b0102d55
commit 63112bfc50
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+5 -2
View File
@@ -109,8 +109,11 @@ class CamoufoxEngine:
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
content_type_lower = content_type.lower()
if 'charset=' in content_type_lower:
encoding = content_type_lower.split('charset=')[-1].split(';')[0].strip().strip('"').strip("'")
if 'utf-8' in encoding:
encoding = 'utf-8'
status_text = res.status_text
# PlayWright API sometimes give empty status text for some reason!