freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: applying the new ruff rules to all files

Browse Source
This commit is contained in:
Karim shoair
2025-09-13 03:22:53 +03:00
parent 60be9dc816
commit 330d03559c
18 changed files with 176 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_html_utils.py
+3 -9
View File
@@ -269,17 +269,13 @@ name2codepoint = {
}
def to_unicode(
text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict"
) -> str:
def to_unicode(text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict") -> str:
"""Return the Unicode representation of a bytes object `text`. If `text`
is already a Unicode object, return it as-is."""
if isinstance(text, str):
return text
if not isinstance(text, (bytes, str)):
raise TypeError(
f"to_unicode must receive bytes or str, got {type(text).__name__}"
)
raise TypeError(f"to_unicode must receive bytes or str, got {type(text).__name__}")
if encoding is None:
encoding = "utf-8"
return text.decode(encoding, errors)
@@ -328,9 +324,7 @@ def _replace_entities(
entity_name = groups["named"]
if entity_name.lower() in keep:
return m.group(0)
number = name2codepoint.get(entity_name) or name2codepoint.get(
entity_name.lower()
)
number = name2codepoint.get(entity_name) or name2codepoint.get(entity_name.lower())
if number is not None:
# Browsers typically
# interpret numeric character references in the 80-9F range as representing the characters mapped
scrapling/core/ai.py
+4 -12
View File
@@ -32,21 +32,13 @@ class ResponseModel(BaseModel):
"""Request's response information structure."""
status: int = Field(description="The status code returned by the website.")
content: list[str] = Field(
description="The content as Markdown/HTML or the text content of the page."
)
url: str = Field(
description="The URL given by the user that resulted in this response."
)
content: list[str] = Field(description="The content as Markdown/HTML or the text content of the page.")
url: str = Field(description="The URL given by the user that resulted in this response.")
def _ContentTranslator(
content: Generator[str, None, None], page: _ScraplingResponse
) -> ResponseModel:
def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:
"""Convert a content generator to a list of ResponseModel objects."""
return ResponseModel(
status=page.status, content=[result for result in content], url=page.url
)
return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
class ScraplingMCPServer:
scrapling/core/custom_types.py
+19 -61
View File
@@ -31,15 +31,11 @@ class TextHandler(str):
__slots__ = ()
def __getitem__(
self, key: SupportsIndex | slice
) -> "TextHandler": # pragma: no cover
def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler": # pragma: no cover
lst = super().__getitem__(key)
return cast(_TextHandlerType, TextHandler(lst))
def split(
self, sep: str = None, maxsplit: SupportsIndex = -1
) -> "TextHandlers": # pragma: no cover
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers": # pragma: no cover
return TextHandlers(
cast(
List[_TextHandlerType],
@@ -50,14 +46,10 @@ class TextHandler(str):
def strip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().strip(chars))
def lstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().lstrip(chars))
def rstrip(
self, chars: str = None
) -> Union[str, "TextHandler"]: # pragma: no cover
def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -66,37 +58,25 @@ class TextHandler(str):
def casefold(self) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().casefold())
def center(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
def center(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().center(width, fillchar))
def expandtabs(
self, tabsize: SupportsIndex = 8
) -> Union[str, "TextHandler"]: # pragma: no cover
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().expandtabs(tabsize))
def format(
self, *args: str, **kwargs: str
) -> Union[str, "TextHandler"]: # pragma: no cover
def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format_map(mapping))
def join(
self, iterable: Iterable[str]
) -> Union[str, "TextHandler"]: # pragma: no cover
def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().join(iterable))
def ljust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
def ljust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().ljust(width, fillchar))
def rjust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]: # pragma: no cover
def rjust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -108,14 +88,10 @@ class TextHandler(str):
def translate(self, table) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().translate(table))
def zfill(
self, width: SupportsIndex
) -> Union[str, "TextHandler"]: # pragma: no cover
def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().zfill(width))
def replace(
self, old: str, new: str, count: SupportsIndex = -1
) -> Union[str, "TextHandler"]:
def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, "TextHandler"]:
return TextHandler(super().replace(old, new, count))
def upper(self) -> Union[str, "TextHandler"]:
@@ -203,11 +179,7 @@ class TextHandler(str):
results = flatten(results)
if not replace_entities:
return TextHandlers(
cast(
List[_TextHandlerType], [TextHandler(string) for string in results]
)
)
return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
return TextHandlers(
cast(
@@ -257,9 +229,7 @@ class TextHandlers(List[TextHandler]):
def __getitem__(self, pos: slice) -> "TextHandlers": # pragma: no cover
pass
def __getitem__(
self, pos: SupportsIndex | slice
) -> Union[TextHandler, "TextHandlers"]:
def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]:
lst = super().__getitem__(pos)
if isinstance(pos, slice):
return TextHandlers(cast(List[_TextHandlerType], lst))
@@ -280,9 +250,7 @@ class TextHandlers(List[TextHandler]):
:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
:param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it
"""
results = [
n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
]
results = [n.re(regex, replace_entities, clean_match, case_sensitive) for n in self]
return TextHandlers(flatten(results))
def re_first(
@@ -330,34 +298,24 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
def __init__(self, mapping=None, **kwargs):
mapping = (
{
key: TextHandler(value) if isinstance(value, str) else value
for key, value in mapping.items()
}
{key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()}
if mapping is not None
else {}
)
if kwargs:
mapping.update(
{
key: TextHandler(value) if isinstance(value, str) else value
for key, value in kwargs.items()
}
{key: TextHandler(value) if isinstance(value, str) else value for key, value in kwargs.items()}
)
# Fastest read-only mapping type
self._data = MappingProxyType(mapping)
def get(
self, key: str, default: Optional[str] = None
) -> Optional[_TextHandlerType]:
def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]:
"""Acts like the standard dictionary `.get()` method"""
return self._data.get(key, default)
def search_values(
self, keyword: str, partial: bool = False
) -> Generator["AttributesHandler", None, None]:
def search_values(self, keyword: str, partial: bool = False) -> Generator["AttributesHandler", None, None]:
"""Search current attributes by values and return a dictionary of each matching item
:param keyword: The keyword to search for in the attribute values
:param partial: If True, the function will search if keyword in each value instead of perfect match
scrapling/core/mixins.py
+6 -28
View File
@@ -5,9 +5,7 @@ class SelectorsGeneration:
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
"""
def __general_selection(
self, selection: str = "css", full_path: bool = False
) -> str:
def __general_selection(self, selection: str = "css", full_path: bool = False) -> str:
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
@@ -18,18 +16,10 @@ class SelectorsGeneration:
if target.parent:
if target.attrib.get("id"):
# id is enough
part = (
f"#{target.attrib['id']}"
if css
else f"[@id='{target.attrib['id']}']"
)
part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']"
selectorPath.append(part)
if not full_path:
return (
" > ".join(reversed(selectorPath))
if css
else "//*" + "/".join(reversed(selectorPath))
)
return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath))
else:
part = f"{target.tag}"
# We won't use classes anymore because I some websites share exact classes between elements
@@ -45,28 +35,16 @@ class SelectorsGeneration:
break
if counter[target.tag] > 1:
part += (
f":nth-of-type({counter[target.tag]})"
if css
else f"[{counter[target.tag]}]"
)
part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]"
selectorPath.append(part)
target = target.parent
if target is None or target.tag == "html":
return (
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
else:
break
return (
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
@property
def generate_css_selector(self) -> str:
scrapling/core/shell.py
+19 -63
View File
@@ -79,9 +79,7 @@ def _CookieParser(cookie_string):
yield key, morsel.value
def _ParseHeaders(
header_lines: List[str], parse_cookies: bool = True
) -> Tuple[Dict[str, str], Dict[str, str]]:
def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Parses headers into separate header and cookie dictionaries."""
header_dict = dict()
cookie_dict = dict()
@@ -93,9 +91,7 @@ def _ParseHeaders(
header_value = ""
header_dict[header_key] = header_value
else:
raise ValueError(
f"Could not parse header without colon: '{header_line}'."
)
raise ValueError(f"Could not parse header without colon: '{header_line}'.")
else:
header_key, header_value = header_line.split(":", 1)
header_key = header_key.strip()
@@ -104,13 +100,9 @@ def _ParseHeaders(
if parse_cookies:
if header_key.lower() == "cookie":
try:
cookie_dict = {
key: value for key, value in _CookieParser(header_value)
}
cookie_dict = {key: value for key, value in _CookieParser(header_value)}
except Exception as e: # pragma: no cover
raise ValueError(
f"Could not parse cookie string from header '{header_value}': {e}"
)
raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")
else:
header_dict[header_key] = header_value
else:
@@ -129,9 +121,7 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no cover
if message:
log.error(f"Scrapling shell exited with status {status}: {message}")
self._print_message(message, stderr)
raise ValueError(
f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}"
)
raise ValueError(f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}")
class CurlParser:
@@ -152,15 +142,11 @@ class CurlParser:
# Data arguments (prioritizing types common from DevTools)
_parser.add_argument("-d", "--data", default=None)
_parser.add_argument(
"--data-raw", default=None
) # Often used by browsers for JSON body
_parser.add_argument("--data-raw", default=None) # Often used by browsers for JSON body
_parser.add_argument("--data-binary", default=None)
# Keep urlencode for completeness, though less common from browser copy/paste
_parser.add_argument("--data-urlencode", action="append", default=[])
_parser.add_argument(
"-G", "--get", action="store_true"
) # Use GET and put data in URL
_parser.add_argument("-G", "--get", action="store_true") # Use GET and put data in URL
_parser.add_argument(
"-b",
@@ -175,9 +161,7 @@ class CurlParser:
# Connection/Security
_parser.add_argument("-k", "--insecure", action="store_true")
_parser.add_argument(
"--compressed", action="store_true"
) # Very common from browsers
_parser.add_argument("--compressed", action="store_true") # Very common from browsers
# Other flags often included but may not map directly to request args
_parser.add_argument("-i", "--include", action="store_true")
@@ -194,9 +178,7 @@ class CurlParser:
clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ")
try:
tokens = shlex_split(
clean_command
) # Split the string using shell-like syntax
tokens = shlex_split(clean_command) # Split the string using shell-like syntax
except ValueError as e: # pragma: no cover
log.error(f"Could not split command line: {e}")
return None
@@ -213,9 +195,7 @@ class CurlParser:
raise
except Exception as e: # pragma: no cover
log.error(
f"An unexpected error occurred during curl arguments parsing: {e}"
)
log.error(f"An unexpected error occurred during curl arguments parsing: {e}")
return None
# --- Determine Method ---
@@ -247,9 +227,7 @@ class CurlParser:
cookies[key] = value
log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
except Exception as e: # pragma: no cover
log.error(
f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}"
)
log.error(f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}")
# --- Process Data Payload ---
params = dict()
@@ -280,9 +258,7 @@ class CurlParser:
try:
data_payload = dict(parse_qsl(combined_data, keep_blank_values=True))
except Exception as e:
log.warning(
f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string."
)
log.warning(f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string.")
data_payload = combined_data
# Check if raw data looks like JSON, prefer 'json' param if so
@@ -303,9 +279,7 @@ class CurlParser:
try:
params.update(dict(parse_qsl(data_payload, keep_blank_values=True)))
except ValueError:
log.warning(
f"Could not parse data '{data_payload}' into GET parameters for -G."
)
log.warning(f"Could not parse data '{data_payload}' into GET parameters for -G.")
if params:
data_payload = None # Clear data as it's moved to params
@@ -314,21 +288,13 @@ class CurlParser:
# --- Process Proxy ---
proxies: Optional[Dict[str, str]] = None
if parsed_args.proxy:
proxy_url = (
f"http://{parsed_args.proxy}"
if "://" not in parsed_args.proxy
else parsed_args.proxy
)
proxy_url = f"http://{parsed_args.proxy}" if "://" not in parsed_args.proxy else parsed_args.proxy
if parsed_args.proxy_user:
user_pass = parsed_args.proxy_user
parts = urlparse(proxy_url)
netloc_parts = parts.netloc.split("@")
netloc = (
f"{user_pass}@{netloc_parts[-1]}"
if len(netloc_parts) > 1
else f"{user_pass}@{parts.netloc}"
)
netloc = f"{user_pass}@{netloc_parts[-1]}" if len(netloc_parts) > 1 else f"{user_pass}@{parts.netloc}"
proxy_url = urlunparse(
(
parts.scheme,
@@ -359,11 +325,7 @@ class CurlParser:
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
if isinstance(curl_command, (Request, str)):
request = (
self.parse(curl_command)
if isinstance(curl_command, str)
else curl_command
)
request = self.parse(curl_command) if isinstance(curl_command, str) else curl_command
# Ensure request parsing was successful before proceeding
if request is None: # pragma: no cover
@@ -386,9 +348,7 @@ class CurlParser:
log.error(f"Error calling Fetcher.{method}: {e}")
return None
else: # pragma: no cover
log.error(
f'Request method "{method}" isn\'t supported by Scrapling yet'
)
log.error(f'Request method "{method}" isn\'t supported by Scrapling yet')
return None
else: # pragma: no cover
@@ -621,18 +581,14 @@ class Convertor:
yield ""
@classmethod
def write_content_to_file(
cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None:
def write_content_to_file(cls, page: Selector, filename: str, css_selector: Optional[str] = None) -> None:
"""Write a Selector's content to a file"""
if not page or not isinstance(page, Selector): # pragma: no cover
raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip():
raise ValueError("Filename must be provided")
elif not filename.endswith((".md", ".html", ".txt")):
raise ValueError(
"Unknown file type: filename must end with '.md', '.html', or '.txt'"
)
raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'")
else:
with open(filename, "w", encoding="utf-8") as f:
extension = filename.split(".")[-1]
scrapling/core/storage.py
+2 -8
View File
@@ -27,11 +27,7 @@ class StorageSystemMixin(ABC): # pragma: no cover
try:
extracted = tld(self.url)
return (
extracted.top_domain_under_public_suffix
or extracted.domain
or default_value
)
return extracted.top_domain_under_public_suffix or extracted.domain or default_value
except AttributeError:
return default_value
@@ -90,9 +86,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
self.connection.execute("PRAGMA journal_mode=WAL")
self.cursor = self.connection.cursor()
self._setup_database()
log.debug(
f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")'
)
log.debug(f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")')
def _setup_database(self) -> None:
self.cursor.execute("""
scrapling/core/translator.py
+6 -18
View File
@@ -89,9 +89,7 @@ class TranslatorMixin:
xpath = super().xpath_element(selector) # type: ignore[safe-super]
return XPathExpr.from_xpath(xpath)
def xpath_pseudo_element(
self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement
) -> OriginalXPathExpr:
def xpath_pseudo_element(self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement) -> OriginalXPathExpr:
"""
Dispatch method that transforms XPath to support the pseudo-element.
"""
@@ -99,31 +97,21 @@ class TranslatorMixin:
method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
method = getattr(self, method_name, None)
if not method: # pragma: no cover
raise ExpressionError(
f"The functional pseudo-element ::{pseudo_element.name}() is unknown"
)
raise ExpressionError(f"The functional pseudo-element ::{pseudo_element.name}() is unknown")
xpath = method(xpath, pseudo_element)
else:
method_name = (
f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
)
method_name = f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
method = getattr(self, method_name, None)
if not method: # pragma: no cover
raise ExpressionError(
f"The pseudo-element ::{pseudo_element} is unknown"
)
raise ExpressionError(f"The pseudo-element ::{pseudo_element} is unknown")
xpath = method(xpath)
return xpath
@staticmethod
def xpath_attr_functional_pseudo_element(
xpath: OriginalXPathExpr, function: FunctionalPseudoElement
) -> XPathExpr:
def xpath_attr_functional_pseudo_element(xpath: OriginalXPathExpr, function: FunctionalPseudoElement) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]): # pragma: no cover
raise ExpressionError(
f"Expected a single string or ident for ::attr(), got {function.arguments!r}"
)
raise ExpressionError(f"Expected a single string or ident for ::attr(), got {function.arguments!r}")
return XPathExpr.from_xpath(xpath, attribute=function.arguments[0].value)
@staticmethod
scrapling/core/utils.py
+5 -21
View File
@@ -24,9 +24,7 @@ def setup_logger():
logger = logging.getLogger("scrapling")
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
formatter = logging.Formatter(fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
@@ -61,11 +59,7 @@ class _StorageTools:
def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:
if not element.attrib:
return {}
return {
k: v.strip()
for k, v in element.attrib.items()
if v and v.strip() and k not in forbidden
}
return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden}
@classmethod
def element_to_dict(cls, element: html.HtmlElement) -> Dict:
@@ -85,17 +79,11 @@ class _StorageTools:
}
)
siblings = [
child.tag for child in parent.iterchildren() if child != element
]
siblings = [child.tag for child in parent.iterchildren() if child != element]
if siblings:
result.update({"siblings": tuple(siblings)})
children = [
child.tag
for child in element.iterchildren()
if not isinstance(child, html_forbidden)
]
children = [child.tag for child in element.iterchildren() if not isinstance(child, html_forbidden)]
if children:
result.update({"children": tuple(children)})
@@ -104,11 +92,7 @@ class _StorageTools:
@classmethod
def _get_element_path(cls, element: html.HtmlElement):
parent = element.getparent()
return tuple(
(element.tag,)
if parent is None
else (cls._get_element_path(parent) + (element.tag,))
)
return tuple((element.tag,) if parent is None else (cls._get_element_path(parent) + (element.tag,)))
@lru_cache(128, typed=True)