test: Rewrite parser tests to a cleaner version and adding more tests
This commit is contained in:
Karim shoair
+90
-122
@@ -1,129 +1,97 @@
|
||||
import unittest
|
||||
import pytest
|
||||
|
||||
from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
|
||||
|
||||
|
||||
class TestPlayWrightFetcher(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.content_type_map = {
|
||||
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome
|
||||
'text/html; charset=UTF-8': 'UTF-8',
|
||||
'text/html; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/html': 'ISO-8859-1',
|
||||
'application/json; charset=UTF-8': 'UTF-8',
|
||||
'application/json': 'utf-8',
|
||||
'text/json': 'utf-8',
|
||||
'application/javascript; charset=UTF-8': 'UTF-8',
|
||||
'application/javascript': 'utf-8',
|
||||
'text/plain; charset=UTF-8': 'UTF-8',
|
||||
'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/plain': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset=UTF-8': 'UTF-8',
|
||||
'application/xhtml+xml': 'utf-8',
|
||||
'text/html; charset=windows-1252': 'windows-1252',
|
||||
'application/json; charset=windows-1252': 'windows-1252',
|
||||
'text/plain; charset=windows-1252': 'windows-1252',
|
||||
'text/html; charset="UTF-8"': 'UTF-8',
|
||||
'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/html; charset="windows-1252"': 'windows-1252',
|
||||
'application/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/json; charset="windows-1252"': 'windows-1252',
|
||||
'text/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="UTF-8"': 'UTF-8',
|
||||
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="windows-1252"': 'windows-1252',
|
||||
'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
|
||||
'text/html; charset="US-ASCII"': 'US-ASCII',
|
||||
'application/json; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/plain; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/html; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/json; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'text/plain; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xml': 'utf-8',
|
||||
'text/xml; charset="UTF-8"': 'UTF-8',
|
||||
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/xml': 'utf-8'
|
||||
}
|
||||
self.status_map = {
|
||||
100: "Continue",
|
||||
101: "Switching Protocols",
|
||||
102: "Processing",
|
||||
103: "Early Hints",
|
||||
200: "OK",
|
||||
201: "Created",
|
||||
202: "Accepted",
|
||||
203: "Non-Authoritative Information",
|
||||
204: "No Content",
|
||||
205: "Reset Content",
|
||||
206: "Partial Content",
|
||||
207: "Multi-Status",
|
||||
208: "Already Reported",
|
||||
226: "IM Used",
|
||||
300: "Multiple Choices",
|
||||
301: "Moved Permanently",
|
||||
302: "Found",
|
||||
303: "See Other",
|
||||
304: "Not Modified",
|
||||
305: "Use Proxy",
|
||||
307: "Temporary Redirect",
|
||||
308: "Permanent Redirect",
|
||||
400: "Bad Request",
|
||||
401: "Unauthorized",
|
||||
402: "Payment Required",
|
||||
403: "Forbidden",
|
||||
404: "Not Found",
|
||||
405: "Method Not Allowed",
|
||||
406: "Not Acceptable",
|
||||
407: "Proxy Authentication Required",
|
||||
408: "Request Timeout",
|
||||
409: "Conflict",
|
||||
410: "Gone",
|
||||
411: "Length Required",
|
||||
412: "Precondition Failed",
|
||||
413: "Payload Too Large",
|
||||
414: "URI Too Long",
|
||||
415: "Unsupported Media Type",
|
||||
416: "Range Not Satisfiable",
|
||||
417: "Expectation Failed",
|
||||
418: "I'm a teapot",
|
||||
421: "Misdirected Request",
|
||||
422: "Unprocessable Entity",
|
||||
423: "Locked",
|
||||
424: "Failed Dependency",
|
||||
425: "Too Early",
|
||||
426: "Upgrade Required",
|
||||
428: "Precondition Required",
|
||||
429: "Too Many Requests",
|
||||
431: "Request Header Fields Too Large",
|
||||
451: "Unavailable For Legal Reasons",
|
||||
500: "Internal Server Error",
|
||||
501: "Not Implemented",
|
||||
502: "Bad Gateway",
|
||||
503: "Service Unavailable",
|
||||
504: "Gateway Timeout",
|
||||
505: "HTTP Version Not Supported",
|
||||
506: "Variant Also Negotiates",
|
||||
507: "Insufficient Storage",
|
||||
508: "Loop Detected",
|
||||
510: "Not Extended",
|
||||
511: "Network Authentication Required"
|
||||
}
|
||||
@pytest.fixture
|
||||
def content_type_map():
|
||||
return {
|
||||
# A map generated by ChatGPT for most possible `content_type` values and the expected outcome
|
||||
'text/html; charset=UTF-8': 'UTF-8',
|
||||
'text/html; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/html': 'ISO-8859-1',
|
||||
'application/json; charset=UTF-8': 'UTF-8',
|
||||
'application/json': 'utf-8',
|
||||
'text/json': 'utf-8',
|
||||
'application/javascript; charset=UTF-8': 'UTF-8',
|
||||
'application/javascript': 'utf-8',
|
||||
'text/plain; charset=UTF-8': 'UTF-8',
|
||||
'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
|
||||
'text/plain': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset=UTF-8': 'UTF-8',
|
||||
'application/xhtml+xml': 'utf-8',
|
||||
'text/html; charset=windows-1252': 'windows-1252',
|
||||
'application/json; charset=windows-1252': 'windows-1252',
|
||||
'text/plain; charset=windows-1252': 'windows-1252',
|
||||
'text/html; charset="UTF-8"': 'UTF-8',
|
||||
'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/html; charset="windows-1252"': 'windows-1252',
|
||||
'application/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/json; charset="windows-1252"': 'windows-1252',
|
||||
'text/json; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="UTF-8"': 'UTF-8',
|
||||
'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="UTF-8"': 'UTF-8',
|
||||
'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/plain; charset="windows-1252"': 'windows-1252',
|
||||
'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
|
||||
'text/html; charset="US-ASCII"': 'US-ASCII',
|
||||
'application/json; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/plain; charset="US-ASCII"': 'US-ASCII',
|
||||
'text/html; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/json; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'text/plain; charset="Shift_JIS"': 'Shift_JIS',
|
||||
'application/xml; charset="UTF-8"': 'UTF-8',
|
||||
'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'application/xml': 'utf-8',
|
||||
'text/xml; charset="UTF-8"': 'UTF-8',
|
||||
'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
|
||||
'text/xml': 'utf-8'
|
||||
}
|
||||
|
||||
def test_parsing_content_type(self):
|
||||
"""Test if parsing different types of content-type returns the expected result"""
|
||||
for header_value, expected_encoding in self.content_type_map.items():
|
||||
self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding)
|
||||
|
||||
def test_parsing_response_status(self):
|
||||
"""Test if using different http responses' status codes returns the expected result"""
|
||||
for status_code, expected_status_text in self.status_map.items():
|
||||
self.assertEqual(StatusText.get(status_code), expected_status_text)
|
||||
@pytest.fixture
|
||||
def status_map():
|
||||
return {
|
||||
100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
|
||||
200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
|
||||
204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
|
||||
208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
|
||||
301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
|
||||
305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
|
||||
400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
|
||||
404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
|
||||
407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
|
||||
410: "Gone", 411: "Length Required", 412: "Precondition Failed",
|
||||
413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
|
||||
416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
|
||||
421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
|
||||
424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
|
||||
428: "Precondition Required", 429: "Too Many Requests",
|
||||
431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
|
||||
500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
|
||||
503: "Service Unavailable", 504: "Gateway Timeout",
|
||||
505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
|
||||
507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
|
||||
511: "Network Authentication Required"
|
||||
}
|
||||
|
||||
self.assertEqual(StatusText.get(1000), "Unknown Status Code")
|
||||
|
||||
def test_parsing_content_type(content_type_map):
|
||||
"""Test if parsing different types of content-type returns the expected result"""
|
||||
for header_value, expected_encoding in content_type_map.items():
|
||||
assert ResponseEncoding.get_value(header_value) == expected_encoding
|
||||
|
||||
|
||||
def test_parsing_response_status(status_map):
|
||||
"""Test if using different http responses' status codes returns the expected result"""
|
||||
for status_code, expected_status_text in status_map.items():
|
||||
assert StatusText.get(status_code) == expected_status_text
|
||||
|
||||
|
||||
def test_unknown_status_code():
|
||||
"""Test handling of an unknown status code"""
|
||||
assert StatusText.get(1000) == "Unknown Status Code"
|
||||
|
||||
Reference in New Issue
Block a user