fix(parser/find_all): Logic issues made conditions used sometimes in a (and) fashion and other times (OR)

Also, the speed boost is 6-14% not 95% as I said first :(
This commit is contained in:
Karim shoair
2025-01-31 00:23:54 +02:00
parent 57090f90a4
commit 1962c11f64
+8 -8
View File
@@ -19,7 +19,7 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
StorageSystemMixin, _StorageTools) StorageSystemMixin, _StorageTools)
from scrapling.core.translator import HTMLTranslator from scrapling.core.translator import HTMLTranslator
from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
is_jsonable, log, lru_cache) is_jsonable, log)
class Adaptor(SelectorsGeneration): class Adaptor(SelectorsGeneration):
@@ -285,7 +285,6 @@ class Adaptor(SelectorsGeneration):
return self.__handle_element(self._root.getparent()) return self.__handle_element(self._root.getparent())
@property @property
@lru_cache(None, True)
def below_elements(self) -> 'Adaptors[Adaptor]': def below_elements(self) -> 'Adaptors[Adaptor]':
"""Return all elements under the current element in the DOM tree""" """Return all elements under the current element in the DOM tree"""
below = self._root.xpath('.//*') below = self._root.xpath('.//*')
@@ -603,12 +602,12 @@ class Adaptor(SelectorsGeneration):
raise TypeError('Nested Iterables are not accepted, only iterables of tag names are accepted') raise TypeError('Nested Iterables are not accepted, only iterables of tag names are accepted')
tags.update(set(arg)) tags.update(set(arg))
elif type(arg) is dict: elif isinstance(arg, dict):
if not all([(type(k) is str and type(v) is str) for k, v in arg.items()]): if not all([(type(k) is str and type(v) is str) for k, v in arg.items()]):
raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted') raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted')
attributes.update(arg) attributes.update(arg)
elif type(arg) is re.Pattern: elif isinstance(arg, re.Pattern):
patterns.add(arg) patterns.add(arg)
elif callable(arg): elif callable(arg):
@@ -629,14 +628,14 @@ class Adaptor(SelectorsGeneration):
attributes[attribute_name] = value attributes[attribute_name] = value
# It's easier and faster to build a selector than traversing the tree # It's easier and faster to build a selector than traversing the tree
tags = tags or [''] tags = tags or ['*']
for tag in tags: for tag in tags:
selector = tag selector = tag
for key, value in attributes.items(): for key, value in attributes.items():
value = value.replace('"', r'\"') # Escape double quotes in user input value = value.replace('"', r'\"') # Escape double quotes in user input
# Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :) # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
selector += '[{}="{}"]'.format(key, value) selector += '[{}="{}"]'.format(key, value)
if selector: if selector != '*':
selectors.append(selector) selectors.append(selector)
if selectors: if selectors:
@@ -650,12 +649,13 @@ class Adaptor(SelectorsGeneration):
for function in functions: for function in functions:
results = results.filter(function) results = results.filter(function)
else: else:
results = results or self.below_elements
for pattern in patterns: for pattern in patterns:
results.extend(self.find_by_regex(pattern, first_match=False)) results = results.filter(lambda e: e.text.re(pattern, check_match=True))
# Collect element if it fulfills passed function otherwise # Collect element if it fulfills passed function otherwise
for function in functions: for function in functions:
results.extend((results or self.below_elements).filter(function)) results = results.filter(function)
return results return results