feat(parser): replacing tldextract with tld library
This might break the adaptive data users have for websites BUT: 1. tld uses ~3.7x less memory during extraction operations (1.5 MB vs 5.7 MB). 2. tld uses ~56% less memory on import (5.2 MB vs 11.9 MB). 3. Zero dependencies (vs 3 for tldextract). In return, it's 30ms slower for extracting 5000 URLs, which is negligible. Also, the type hints aren't always accurate, but it's fine; I corrected them.
This commit is contained in:
@@ -8,7 +8,7 @@ from orjson import dumps, loads
|
||||
from lxml.html import HtmlElement
|
||||
|
||||
from scrapling.core.utils import _StorageTools, log
|
||||
from scrapling.core._types import Dict, Optional, Any
|
||||
from scrapling.core._types import Dict, Optional, Any, cast
|
||||
|
||||
|
||||
class StorageSystemMixin(ABC): # pragma: no cover
|
||||
@@ -17,18 +17,24 @@ class StorageSystemMixin(ABC): # pragma: no cover
|
||||
"""
|
||||
:param url: URL of the website we are working on to separate it from other websites data
|
||||
"""
|
||||
self.url = url
|
||||
# Make the url in lowercase to handle this edge case until it's updated: https://github.com/barseghyanartur/tld/issues/124
|
||||
self.url = url.lower() if (url and isinstance(url, str)) else None
|
||||
|
||||
@lru_cache(64, typed=True)
|
||||
def _get_base_url(self, default_value: str = "default") -> str:
|
||||
if not self.url or not isinstance(self.url, str):
|
||||
if not self.url:
|
||||
return default_value
|
||||
|
||||
try:
|
||||
from tldextract import extract as tld
|
||||
from tld import get_tld, Result
|
||||
|
||||
extracted = tld(self.url)
|
||||
return extracted.top_domain_under_public_suffix or extracted.domain or default_value
|
||||
# Fixing the inaccurate return type hint in `get_tld`
|
||||
extracted: Result | None = cast(
|
||||
Result, get_tld(self.url, as_object=True, fail_silently=True, fix_protocol=True)
|
||||
)
|
||||
if not extracted:
|
||||
return default_value
|
||||
return extracted.fld or extracted.domain or default_value
|
||||
except AttributeError:
|
||||
return default_value
|
||||
|
||||
|
||||
Reference in New Issue
Block a user