refactor: cleaner code for the fetcher's lazy loader
This commit is contained in:
+15
-36
@@ -3,45 +3,24 @@ __version__ = "0.3-beta"
|
||||
__copyright__ = "Copyright (c) 2024 Karim Shoair"
|
||||
|
||||
|
||||
# A lightweight approach to create lazy loader for each import for backward compatibility
|
||||
# A lightweight approach to create a lazy loader for each import for backward compatibility
|
||||
# This will reduces initial memory footprint significantly (only loads what's used)
|
||||
def __getattr__(name):
|
||||
if name == "Fetcher":
|
||||
from scrapling.fetchers import Fetcher as cls
|
||||
lazy_imports = {
|
||||
"Fetcher": ("scrapling.fetchers", "Fetcher"),
|
||||
"Selector": ("scrapling.parser", "Selector"),
|
||||
"Selectors": ("scrapling.parser", "Selectors"),
|
||||
"AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
|
||||
"TextHandler": ("scrapling.core.custom_types", "TextHandler"),
|
||||
"AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
|
||||
"StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
|
||||
"DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
|
||||
}
|
||||
|
||||
return cls
|
||||
elif name == "Selector":
|
||||
from scrapling.parser import Selector as cls
|
||||
|
||||
return cls
|
||||
elif name == "Selectors":
|
||||
from scrapling.parser import Selectors as cls
|
||||
|
||||
return cls
|
||||
elif name == "AttributesHandler":
|
||||
from scrapling.core.custom_types import AttributesHandler as cls
|
||||
|
||||
return cls
|
||||
elif name == "TextHandler":
|
||||
from scrapling.core.custom_types import TextHandler as cls
|
||||
|
||||
return cls
|
||||
elif name == "AsyncFetcher":
|
||||
from scrapling.fetchers import AsyncFetcher as cls
|
||||
|
||||
return cls
|
||||
elif name == "StealthyFetcher":
|
||||
from scrapling.fetchers import StealthyFetcher as cls
|
||||
|
||||
return cls
|
||||
elif name == "DynamicFetcher":
|
||||
from scrapling.fetchers import DynamicFetcher as cls
|
||||
|
||||
return cls
|
||||
elif name == "CustomFetcher":
|
||||
from scrapling.fetchers import CustomFetcher as cls
|
||||
|
||||
return cls
|
||||
if name in lazy_imports:
|
||||
module_path, class_name = lazy_imports[name]
|
||||
module = __import__(module_path, fromlist=[class_name])
|
||||
return getattr(module, class_name)
|
||||
else:
|
||||
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user