refactor: cleaner code for the fetcher's lazy loader

This commit is contained in:
Karim shoair
2025-07-29 22:43:12 +03:00
parent 54cba6db45
commit 3d07a1533e
+15 -36
View File
@@ -3,45 +3,24 @@ __version__ = "0.3-beta"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
# A lightweight approach to create lazy loader for each import for backward compatibility
# A lightweight approach to create a lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name):
if name == "Fetcher":
from scrapling.fetchers import Fetcher as cls
lazy_imports = {
"Fetcher": ("scrapling.fetchers", "Fetcher"),
"Selector": ("scrapling.parser", "Selector"),
"Selectors": ("scrapling.parser", "Selectors"),
"AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
"TextHandler": ("scrapling.core.custom_types", "TextHandler"),
"AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
"StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
"DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
}
return cls
elif name == "Selector":
from scrapling.parser import Selector as cls
return cls
elif name == "Selectors":
from scrapling.parser import Selectors as cls
return cls
elif name == "AttributesHandler":
from scrapling.core.custom_types import AttributesHandler as cls
return cls
elif name == "TextHandler":
from scrapling.core.custom_types import TextHandler as cls
return cls
elif name == "AsyncFetcher":
from scrapling.fetchers import AsyncFetcher as cls
return cls
elif name == "StealthyFetcher":
from scrapling.fetchers import StealthyFetcher as cls
return cls
elif name == "DynamicFetcher":
from scrapling.fetchers import DynamicFetcher as cls
return cls
elif name == "CustomFetcher":
from scrapling.fetchers import CustomFetcher as cls
return cls
if name in lazy_imports:
module_path, class_name = lazy_imports[name]
module = __import__(module_path, fromlist=[class_name])
return getattr(module, class_name)
else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")