From 3068bb135621589276e4be90edf2240fd7fa66f3 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 14 Jun 2025 18:56:31 +0300 Subject: [PATCH] build: update deps --- pyproject.toml | 4 ++-- scrapling/core/storage_adaptors.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d70a8b0..77e450a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,8 +59,8 @@ dependencies = [ "IPython", "click", "orjson>=3", - "tldextract", - "curl_cffi>=0.11.1", + "tldextract>=5.3.0", + "curl_cffi>=0.11.3", "playwright>=1.52.0", "rebrowser-playwright>=1.52.0", "camoufox[geoip]>=0.4.11", diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.py index d13b549..cabbde5 100644 --- a/scrapling/core/storage_adaptors.py +++ b/scrapling/core/storage_adaptors.py @@ -26,7 +26,11 @@ class StorageSystemMixin(ABC): try: extracted = tld(self.url) - return extracted.registered_domain or extracted.domain or default_value + return ( + extracted.top_domain_under_public_suffix + or extracted.domain + or default_value + ) except AttributeError: return default_value