Adding documentation to the base fetcher...
+ Configure logger from here. + Enable auto_match by default.
This commit is contained in:
@@ -6,6 +6,7 @@ import logging
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
from scrapling.parser import Adaptor, SQLiteStorageSystem
|
from scrapling.parser import Adaptor, SQLiteStorageSystem
|
||||||
|
from scrapling.core.utils import setup_basic_logging
|
||||||
from scrapling.core._types import Any, List, Type, Union, Optional, Dict
|
from scrapling.core._types import Any, List, Type, Union, Optional, Dict
|
||||||
|
|
||||||
|
|
||||||
@@ -36,15 +37,23 @@ class Response:
|
|||||||
|
|
||||||
class BaseFetcher:
|
class BaseFetcher:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
|
||||||
# Adaptor class parameters
|
storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
|
||||||
huge_tree: bool = True,
|
|
||||||
keep_comments: Optional[bool] = False,
|
|
||||||
auto_match: Optional[bool] = False,
|
|
||||||
storage: Any = SQLiteStorageSystem,
|
|
||||||
storage_args: Optional[Dict] = None,
|
|
||||||
debug: Optional[bool] = True,
|
|
||||||
):
|
):
|
||||||
|
"""Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
|
||||||
|
are detected and passed automatically from the Fetcher based on the response for accessibility.
|
||||||
|
|
||||||
|
:param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
|
||||||
|
libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
|
||||||
|
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
|
||||||
|
:param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
|
||||||
|
priority over all auto-match related arguments/functions in the class.
|
||||||
|
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
|
||||||
|
:param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
|
||||||
|
If empty, default values will be used.
|
||||||
|
:param debug: Enable debug mode
|
||||||
|
"""
|
||||||
|
# Adaptor class parameters
|
||||||
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
|
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
|
||||||
self.adaptor_arguments = dict(
|
self.adaptor_arguments = dict(
|
||||||
huge_tree=huge_tree,
|
huge_tree=huge_tree,
|
||||||
@@ -54,6 +63,8 @@ class BaseFetcher:
|
|||||||
storage_args=storage_args,
|
storage_args=storage_args,
|
||||||
debug=debug,
|
debug=debug,
|
||||||
)
|
)
|
||||||
|
# If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
|
||||||
|
setup_basic_logging(level='debug' if debug else 'info')
|
||||||
|
|
||||||
|
|
||||||
def check_if_engine_usable(engine):
|
def check_if_engine_usable(engine):
|
||||||
|
|||||||
Reference in New Issue
Block a user