freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13 from andyfcx/main

Browse Source 
fix: make sure to default utf-8 as encoding and strip unwanted single or double quotes
This commit is contained in:
Karim shoair
2024-11-21 11:21:45 +02:00
committed by GitHub
parent e9b0102d55 63112bfc50
commit 18eb96adb1
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/camo.py
+5 -2
View File
@@ -109,8 +109,11 @@ class CamoufoxEngine:
content_type = res.headers.get('content-type', '')
# Parse charset from content-type
encoding = 'utf-8' # default encoding
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
content_type_lower = content_type.lower()
if 'charset=' in content_type_lower:
encoding = content_type_lower.split('charset=')[-1].split(';')[0].strip().strip('"').strip("'")
if 'utf-8' in encoding:
encoding = 'utf-8'
status_text = res.status_text
# PlayWright API sometimes give empty status text for some reason!