freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fetchers/test_utils.py
+105 -64
View File
@@ -7,76 +7,117 @@ from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
def content_type_map():
return {
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome
'text/html; charset=UTF-8': 'UTF-8',
'text/html; charset=ISO-8859-1': 'ISO-8859-1',
'text/html': 'ISO-8859-1',
'application/json; charset=UTF-8': 'UTF-8',
'application/json': 'utf-8',
'text/json': 'utf-8',
'application/javascript; charset=UTF-8': 'UTF-8',
'application/javascript': 'utf-8',
'text/plain; charset=UTF-8': 'UTF-8',
'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
'text/plain': 'ISO-8859-1',
'application/xhtml+xml; charset=UTF-8': 'UTF-8',
'application/xhtml+xml': 'utf-8',
'text/html; charset=windows-1252': 'windows-1252',
'application/json; charset=windows-1252': 'windows-1252',
'text/plain; charset=windows-1252': 'windows-1252',
'text/html; charset="UTF-8"': 'UTF-8',
'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
'text/html; charset="windows-1252"': 'windows-1252',
'application/json; charset="UTF-8"': 'UTF-8',
'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
'application/json; charset="windows-1252"': 'windows-1252',
'text/json; charset="UTF-8"': 'UTF-8',
'application/javascript; charset="UTF-8"': 'UTF-8',
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
'text/plain; charset="UTF-8"': 'UTF-8',
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
'text/plain; charset="windows-1252"': 'windows-1252',
'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
'text/html; charset="US-ASCII"': 'US-ASCII',
'application/json; charset="US-ASCII"': 'US-ASCII',
'text/plain; charset="US-ASCII"': 'US-ASCII',
'text/html; charset="Shift_JIS"': 'Shift_JIS',
'application/json; charset="Shift_JIS"': 'Shift_JIS',
'text/plain; charset="Shift_JIS"': 'Shift_JIS',
'application/xml; charset="UTF-8"': 'UTF-8',
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
'application/xml': 'utf-8',
'text/xml; charset="UTF-8"': 'UTF-8',
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
'text/xml': 'utf-8'
"text/html; charset=UTF-8": "UTF-8",
"text/html; charset=ISO-8859-1": "ISO-8859-1",
"text/html": "ISO-8859-1",
"application/json; charset=UTF-8": "UTF-8",
"application/json": "utf-8",
"text/json": "utf-8",
"application/javascript; charset=UTF-8": "UTF-8",
"application/javascript": "utf-8",
"text/plain; charset=UTF-8": "UTF-8",
"text/plain; charset=ISO-8859-1": "ISO-8859-1",
"text/plain": "ISO-8859-1",
"application/xhtml+xml; charset=UTF-8": "UTF-8",
"application/xhtml+xml": "utf-8",
"text/html; charset=windows-1252": "windows-1252",
"application/json; charset=windows-1252": "windows-1252",
"text/plain; charset=windows-1252": "windows-1252",
'text/html; charset="UTF-8"': "UTF-8",
'text/html; charset="ISO-8859-1"': "ISO-8859-1",
'text/html; charset="windows-1252"': "windows-1252",
'application/json; charset="UTF-8"': "UTF-8",
'application/json; charset="ISO-8859-1"': "ISO-8859-1",
'application/json; charset="windows-1252"': "windows-1252",
'text/json; charset="UTF-8"': "UTF-8",
'application/javascript; charset="UTF-8"': "UTF-8",
'application/javascript; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="UTF-8"': "UTF-8",
'text/plain; charset="ISO-8859-1"': "ISO-8859-1",
'text/plain; charset="windows-1252"': "windows-1252",
'application/xhtml+xml; charset="UTF-8"': "UTF-8",
'application/xhtml+xml; charset="ISO-8859-1"': "ISO-8859-1",
'application/xhtml+xml; charset="windows-1252"': "windows-1252",
'text/html; charset="US-ASCII"': "US-ASCII",
'application/json; charset="US-ASCII"': "US-ASCII",
'text/plain; charset="US-ASCII"': "US-ASCII",
'text/html; charset="Shift_JIS"': "Shift_JIS",
'application/json; charset="Shift_JIS"': "Shift_JIS",
'text/plain; charset="Shift_JIS"': "Shift_JIS",
'application/xml; charset="UTF-8"': "UTF-8",
'application/xml; charset="ISO-8859-1"': "ISO-8859-1",
"application/xml": "utf-8",
'text/xml; charset="UTF-8"': "UTF-8",
'text/xml; charset="ISO-8859-1"': "ISO-8859-1",
"text/xml": "utf-8",
}
@pytest.fixture
def status_map():
return {
100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
410: "Gone", 411: "Length Required", 412: "Precondition Failed",
413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
428: "Precondition Required", 429: "Too Many Requests",
431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
503: "Service Unavailable", 504: "Gateway Timeout",
505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
511: "Network Authentication Required"
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
}