From f33ff247dd02d180739b7cd7924bb753e7c1e34a Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 23 Nov 2024 15:55:21 +0200 Subject: [PATCH] Adding tests for response encoding and response status text --- tests/fetchers/test_utils.py | 129 +++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tests/fetchers/test_utils.py diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py new file mode 100644 index 0000000..0392388 --- /dev/null +++ b/tests/fetchers/test_utils.py @@ -0,0 +1,129 @@ +import unittest + +from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText + + +class TestPlayWrightFetcher(unittest.TestCase): + def setUp(self): + self.content_type_map = { + # A map generated by ChatGPT for most possible `content_type` values and the expected outcome + 'text/html; charset=UTF-8': 'UTF-8', + 'text/html; charset=ISO-8859-1': 'ISO-8859-1', + 'text/html': 'ISO-8859-1', + 'application/json; charset=UTF-8': 'UTF-8', + 'application/json': 'utf-8', + 'text/json': 'utf-8', + 'application/javascript; charset=UTF-8': 'UTF-8', + 'application/javascript': 'utf-8', + 'text/plain; charset=UTF-8': 'UTF-8', + 'text/plain; charset=ISO-8859-1': 'ISO-8859-1', + 'text/plain': 'ISO-8859-1', + 'application/xhtml+xml; charset=UTF-8': 'UTF-8', + 'application/xhtml+xml': 'utf-8', + 'text/html; charset=windows-1252': 'windows-1252', + 'application/json; charset=windows-1252': 'windows-1252', + 'text/plain; charset=windows-1252': 'windows-1252', + 'text/html; charset="UTF-8"': 'UTF-8', + 'text/html; charset="ISO-8859-1"': 'ISO-8859-1', + 'text/html; charset="windows-1252"': 'windows-1252', + 'application/json; charset="UTF-8"': 'UTF-8', + 'application/json; charset="ISO-8859-1"': 'ISO-8859-1', + 'application/json; charset="windows-1252"': 'windows-1252', + 'text/json; charset="UTF-8"': 'UTF-8', + 'application/javascript; charset="UTF-8"': 'UTF-8', + 'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1', + 'text/plain; charset="UTF-8"': 'UTF-8', + 'text/plain; charset="ISO-8859-1"': 'ISO-8859-1', + 'text/plain; charset="windows-1252"': 'windows-1252', + 'application/xhtml+xml; charset="UTF-8"': 'UTF-8', + 'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1', + 'application/xhtml+xml; charset="windows-1252"': 'windows-1252', + 'text/html; charset="US-ASCII"': 'US-ASCII', + 'application/json; charset="US-ASCII"': 'US-ASCII', + 'text/plain; charset="US-ASCII"': 'US-ASCII', + 'text/html; charset="Shift_JIS"': 'Shift_JIS', + 'application/json; charset="Shift_JIS"': 'Shift_JIS', + 'text/plain; charset="Shift_JIS"': 'Shift_JIS', + 'application/xml; charset="UTF-8"': 'UTF-8', + 'application/xml; charset="ISO-8859-1"': 'ISO-8859-1', + 'application/xml': 'utf-8', + 'text/xml; charset="UTF-8"': 'UTF-8', + 'text/xml; charset="ISO-8859-1"': 'ISO-8859-1', + 'text/xml': 'utf-8' + } + self.status_map = { + 100: "Continue", + 101: "Switching Protocols", + 102: "Processing", + 103: "Early Hints", + 200: "OK", + 201: "Created", + 202: "Accepted", + 203: "Non-Authoritative Information", + 204: "No Content", + 205: "Reset Content", + 206: "Partial Content", + 207: "Multi-Status", + 208: "Already Reported", + 226: "IM Used", + 300: "Multiple Choices", + 301: "Moved Permanently", + 302: "Found", + 303: "See Other", + 304: "Not Modified", + 305: "Use Proxy", + 307: "Temporary Redirect", + 308: "Permanent Redirect", + 400: "Bad Request", + 401: "Unauthorized", + 402: "Payment Required", + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 406: "Not Acceptable", + 407: "Proxy Authentication Required", + 408: "Request Timeout", + 409: "Conflict", + 410: "Gone", + 411: "Length Required", + 412: "Precondition Failed", + 413: "Payload Too Large", + 414: "URI Too Long", + 415: "Unsupported Media Type", + 416: "Range Not Satisfiable", + 417: "Expectation Failed", + 418: "I'm a teapot", + 421: "Misdirected Request", + 422: "Unprocessable Entity", + 423: "Locked", + 424: "Failed Dependency", + 425: "Too Early", + 426: "Upgrade Required", + 428: "Precondition Required", + 429: "Too Many Requests", + 431: "Request Header Fields Too Large", + 451: "Unavailable For Legal Reasons", + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", + 503: "Service Unavailable", + 504: "Gateway Timeout", + 505: "HTTP Version Not Supported", + 506: "Variant Also Negotiates", + 507: "Insufficient Storage", + 508: "Loop Detected", + 510: "Not Extended", + 511: "Network Authentication Required" + } + + def test_parsing_content_type(self): + """Test if parsing different types of content-type returns the expected result""" + for header_value, expected_encoding in self.content_type_map.items(): + self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding) + + def test_parsing_response_status(self): + """Test if using different http responses' status codes returns the expected result""" + for status_code, expected_status_text in self.status_map.items(): + self.assertEqual(StatusText.get(status_code), expected_status_text) + + self.assertEqual(1000, "Unknown Status Code")