refactor(parser): Multiple optimizations and fixes
This commit is contained in:
+32
-22
@@ -59,6 +59,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
keep_comments: Optional[bool] = False,
|
keep_comments: Optional[bool] = False,
|
||||||
keep_cdata: Optional[bool] = False,
|
keep_cdata: Optional[bool] = False,
|
||||||
auto_match: Optional[bool] = False,
|
auto_match: Optional[bool] = False,
|
||||||
|
_storage: object = None,
|
||||||
storage: Any = SQLiteStorageSystem,
|
storage: Any = SQLiteStorageSystem,
|
||||||
storage_args: Optional[Dict] = None,
|
storage_args: Optional[Dict] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
@@ -136,25 +137,28 @@ class Adaptor(SelectorsGeneration):
|
|||||||
self.__auto_match_enabled = auto_match
|
self.__auto_match_enabled = auto_match
|
||||||
|
|
||||||
if self.__auto_match_enabled:
|
if self.__auto_match_enabled:
|
||||||
if not storage_args:
|
if _storage is not None:
|
||||||
storage_args = {
|
self._storage = _storage
|
||||||
"storage_file": os.path.join(
|
else:
|
||||||
os.path.dirname(__file__), "elements_storage.db"
|
if not storage_args:
|
||||||
),
|
storage_args = {
|
||||||
"url": url,
|
"storage_file": os.path.join(
|
||||||
}
|
os.path.dirname(__file__), "elements_storage.db"
|
||||||
|
),
|
||||||
|
"url": url,
|
||||||
|
}
|
||||||
|
|
||||||
if not hasattr(storage, "__wrapped__"):
|
if not hasattr(storage, "__wrapped__"):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Storage class must be wrapped with lru_cache decorator, see docs for info"
|
"Storage class must be wrapped with lru_cache decorator, see docs for info"
|
||||||
)
|
)
|
||||||
|
|
||||||
if not issubclass(storage.__wrapped__, StorageSystemMixin):
|
if not issubclass(storage.__wrapped__, StorageSystemMixin):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Storage system must be inherited from class `StorageSystemMixin`"
|
"Storage system must be inherited from class `StorageSystemMixin`"
|
||||||
)
|
)
|
||||||
|
|
||||||
self._storage = storage(**storage_args)
|
self._storage = storage(**storage_args)
|
||||||
|
|
||||||
self.__keep_comments = keep_comments
|
self.__keep_comments = keep_comments
|
||||||
self.__keep_cdata = keep_cdata
|
self.__keep_cdata = keep_cdata
|
||||||
@@ -186,6 +190,8 @@ class Adaptor(SelectorsGeneration):
|
|||||||
if hasattr(self, "_storage") and self._storage:
|
if hasattr(self, "_storage") and self._storage:
|
||||||
try:
|
try:
|
||||||
self._storage.close()
|
self._storage.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
finally:
|
finally:
|
||||||
self._storage = None
|
self._storage = None
|
||||||
|
|
||||||
@@ -214,13 +220,15 @@ class Adaptor(SelectorsGeneration):
|
|||||||
|
|
||||||
def __element_convertor(self, element: html.HtmlElement) -> "Adaptor":
|
def __element_convertor(self, element: html.HtmlElement) -> "Adaptor":
|
||||||
"""Used internally to convert a single HtmlElement to Adaptor directly without checks"""
|
"""Used internally to convert a single HtmlElement to Adaptor directly without checks"""
|
||||||
|
db_instance = (
|
||||||
|
self._storage if (hasattr(self, "_storage") and self._storage) else None
|
||||||
|
)
|
||||||
return Adaptor(
|
return Adaptor(
|
||||||
root=element,
|
root=element,
|
||||||
text="",
|
|
||||||
body=b"", # Since the root argument is provided, both `text` and `body` will be ignored, so this is just a filler
|
|
||||||
url=self.url,
|
url=self.url,
|
||||||
encoding=self.encoding,
|
encoding=self.encoding,
|
||||||
auto_match=self.__auto_match_enabled,
|
auto_match=self.__auto_match_enabled,
|
||||||
|
_storage=db_instance, # Reuse existing storage if it exists otherwise it won't be checked if `auto_match` is turned off
|
||||||
keep_comments=self.__keep_comments,
|
keep_comments=self.__keep_comments,
|
||||||
keep_cdata=self.__keep_cdata,
|
keep_cdata=self.__keep_cdata,
|
||||||
huge_tree=self.__huge_tree_enabled,
|
huge_tree=self.__huge_tree_enabled,
|
||||||
@@ -630,8 +638,10 @@ class Adaptor(SelectorsGeneration):
|
|||||||
except (
|
except (
|
||||||
SelectorError,
|
SelectorError,
|
||||||
SelectorSyntaxError,
|
SelectorSyntaxError,
|
||||||
):
|
) as e:
|
||||||
raise SelectorSyntaxError(f"Invalid CSS selector: {selector}")
|
raise SelectorSyntaxError(
|
||||||
|
f"Invalid CSS selector '{selector}': {str(e)}"
|
||||||
|
) from e
|
||||||
|
|
||||||
def xpath(
|
def xpath(
|
||||||
self,
|
self,
|
||||||
@@ -700,8 +710,8 @@ class Adaptor(SelectorsGeneration):
|
|||||||
SelectorSyntaxError,
|
SelectorSyntaxError,
|
||||||
etree.XPathError,
|
etree.XPathError,
|
||||||
etree.XPathEvalError,
|
etree.XPathEvalError,
|
||||||
):
|
) as e:
|
||||||
raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")
|
raise SelectorSyntaxError(f"Invalid XPath selector: {selector}") from e
|
||||||
|
|
||||||
def find_all(
|
def find_all(
|
||||||
self,
|
self,
|
||||||
|
|||||||
Reference in New Issue
Block a user