Karim shoair
145c03daff
- Moved most of the parser functions/files to the core package. - Converted tools file to a package and made separate files for similar functions. - Now all fetcher engines return a Response object - Instead of selecting an engine to use and passing config to it, we have separate fetcher classes so the user can choose what to use while importing. - I added a new custom fetcher so the user can create and use an engine. - More...
66 lines
2.1 KiB
Python
66 lines
2.1 KiB
Python
"""
|
|
Functions related to generating headers and fingerprints generally
|
|
"""
|
|
|
|
import platform
|
|
|
|
from tldextract import extract
|
|
from browserforge.fingerprints import FingerprintGenerator
|
|
from browserforge.headers import HeaderGenerator, Browser
|
|
|
|
|
|
def generate_convincing_referer(url):
|
|
"""
|
|
Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
|
|
|
|
>>> generate_convincing_referer('https://www.somewebsite.com/blah')
|
|
'https://www.google.com/search?q=somewebsite'
|
|
|
|
:param url: The URL you are about to fetch.
|
|
:return:
|
|
"""
|
|
website_name = extract(url).domain
|
|
return f'https://www.google.com/search?q={website_name}'
|
|
|
|
|
|
def get_os_name():
|
|
# Get the OS name in the same format needed for browserforge
|
|
os_name = platform.system()
|
|
return {
|
|
'Linux': 'linux',
|
|
'Darwin': 'macos',
|
|
'Windows': 'windows',
|
|
# For the future? because why not
|
|
'iOS': 'ios',
|
|
}.get(os_name)
|
|
|
|
|
|
def generate_suitable_fingerprint():
|
|
# This would be for Browserforge playwright injector
|
|
os_name = get_os_name()
|
|
return FingerprintGenerator(
|
|
browser=[Browser(name='chrome', min_version=128)],
|
|
os=os_name, # None is ignored
|
|
device='desktop'
|
|
).generate()
|
|
|
|
|
|
def generate_headers(browser_mode=False):
|
|
if browser_mode:
|
|
# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
|
|
# So we don't raise any inconsistency red flags while websites fingerprinting us
|
|
os_name = get_os_name()
|
|
return HeaderGenerator(
|
|
browser=[Browser(name='chrome', min_version=128)],
|
|
os=os_name, # None is ignored
|
|
device='desktop'
|
|
).generate()
|
|
else:
|
|
# Here it's used for normal requests that aren't done through browsers so we can take it lightly
|
|
browsers = [
|
|
Browser(name='chrome', min_version=120),
|
|
Browser(name='firefox', min_version=120),
|
|
Browser(name='edge', min_version=120),
|
|
]
|
|
return HeaderGenerator(browser=browsers, device='desktop').generate()
|