From 5f4ec3ba3a5c974a4f33f1e9752cb00f93773dea Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Feb 2025 23:14:59 +0200 Subject: [PATCH 1/4] fix(Fetcher): make `stealthy_headers` don't overwrite user headers Fix for #39 --- scrapling/engines/static.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index a939c46..a58aa39 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -42,16 +42,19 @@ class StaticEngine: :return: A dictionary of the new headers. """ headers = headers or {} - - # Validate headers - if not headers.get('user-agent') and not headers.get('User-Agent'): - headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent') - log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") + headers_keys = set(map(str.lower, headers.keys())) if self.stealth: extra_headers = generate_headers(browser_mode=False) + # Don't overwrite user supplied headers + extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys} headers.update(extra_headers) - headers.update({'referer': generate_convincing_referer(self.url)}) + if 'referer' not in headers_keys: + headers.update({'referer': generate_convincing_referer(self.url)}) + + elif 'user-agent' not in headers_keys: + headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent') + log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.") return headers From c596a139015e6a17e456c43e94096cda7cddb385 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Feb 2025 23:54:23 +0200 Subject: [PATCH 2/4] docs: reflect updates on README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cf931f2..f299b3b 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,8 @@ This class is built on top of [httpx](https://www.python-httpx.org/) with additi For all methods, you have `stealthy_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default. You can also set the number of retries with the argument `retries` for all methods and this will make httpx retry requests if it failed for any reason. The default number of retries for all `Fetcher` methods is 3. +> Hence: All headers generated by `stealthy_headers` argument can be overwritten by you through the `headers` argument + You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods in this format `http://username:password@localhost:8030` ```python >> page = Fetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True) From dc702d2d7c3a1b59e1f47a1015741a9200a7df33 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Tue, 25 Feb 2025 23:57:31 +0200 Subject: [PATCH 3/4] build: pumping version up --- scrapling/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapling/__init__.py b/scrapling/__init__.py index a184e7b..c6908db 100644 --- a/scrapling/__init__.py +++ b/scrapling/__init__.py @@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher, from scrapling.parser import Adaptor, Adaptors __author__ = "Karim Shoair (karim.shoair@pm.me)" -__version__ = "0.2.94" +__version__ = "0.2.95" __copyright__ = "Copyright (c) 2024 Karim Shoair" diff --git a/setup.cfg b/setup.cfg index cb144c1..84cb519 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scrapling -version = 0.2.94 +version = 0.2.95 author = Karim Shoair author_email = karim.shoair@pm.me description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python. diff --git a/setup.py b/setup.py index 6eb2534..6b5c647 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", - version="0.2.94", + version="0.2.95", description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It simplifies the process of extracting data from websites, even when they undergo structural changes, and offers impressive speed improvements over many popular scraping tools.""", From 1dc79d449b14bcfedc31519252254ee6acb88ff6 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 26 Feb 2025 00:19:54 +0200 Subject: [PATCH 4/4] docs: update library description --- setup.cfg | 2 +- setup.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 84cb519..ad4de36 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,6 @@ name = scrapling version = 0.2.95 author = Karim Shoair author_email = karim.shoair@pm.me -description = Scrapling is an undetectable, powerful, flexible, adaptive, and high-performance web scraping library for Python. +description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! license = BSD home_page = https://github.com/D4Vinci/Scrapling \ No newline at end of file diff --git a/setup.py b/setup.py index 6b5c647..81afabd 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,8 @@ with open("README.md", "r", encoding="utf-8") as fh: setup( name="scrapling", version="0.2.95", - description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It - simplifies the process of extracting data from websites, even when they undergo structural changes, and offers - impressive speed improvements over many popular scraping tools.""", + description="""Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications, + it simplifies web scraping, even when websites' design changes, while providing impressive speed that surpasses almost all alternatives.""", long_description=long_description, long_description_content_type="text/markdown", author="Karim Shoair",