fix(importing): Lazy Loading submodules to lower memory usage on importing
Load what you use instead of all submodules together
This commit is contained in:
+34
-5
@@ -1,12 +1,41 @@
|
|||||||
# Declare top-level shortcuts
|
|
||||||
from scrapling.core.custom_types import AttributesHandler, TextHandler
|
|
||||||
from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
|
|
||||||
PlayWrightFetcher, StealthyFetcher)
|
|
||||||
from scrapling.parser import Adaptor, Adaptors
|
|
||||||
|
|
||||||
__author__ = "Karim Shoair (karim.shoair@pm.me)"
|
__author__ = "Karim Shoair (karim.shoair@pm.me)"
|
||||||
__version__ = "0.2.97"
|
__version__ = "0.2.97"
|
||||||
__copyright__ = "Copyright (c) 2024 Karim Shoair"
|
__copyright__ = "Copyright (c) 2024 Karim Shoair"
|
||||||
|
|
||||||
|
|
||||||
|
# A lightweight approach to create lazy loader for each import for backward compatibility
|
||||||
|
# This will reduces initial memory footprint significantly (only loads what's used)
|
||||||
|
def __getattr__(name):
|
||||||
|
if name == 'Fetcher':
|
||||||
|
from scrapling.fetchers import Fetcher as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'Adaptor':
|
||||||
|
from scrapling.parser import Adaptor as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'Adaptors':
|
||||||
|
from scrapling.parser import Adaptors as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'AttributesHandler':
|
||||||
|
from scrapling.core.custom_types import AttributesHandler as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'TextHandler':
|
||||||
|
from scrapling.core.custom_types import TextHandler as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'AsyncFetcher':
|
||||||
|
from scrapling.fetchers import AsyncFetcher as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'StealthyFetcher':
|
||||||
|
from scrapling.fetchers import StealthyFetcher as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'PlayWrightFetcher':
|
||||||
|
from scrapling.fetchers import PlayWrightFetcher as cls
|
||||||
|
return cls
|
||||||
|
elif name == 'CustomFetcher':
|
||||||
|
from scrapling.fetchers import CustomFetcher as cls
|
||||||
|
return cls
|
||||||
|
else:
|
||||||
|
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']
|
__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']
|
||||||
|
|||||||
Reference in New Issue
Block a user