From 8205e51306a17b137344da2a675d229a75f5e204 Mon Sep 17 00:00:00 2001 From: Ms2ger Date: Mon, 24 Sep 2018 13:42:16 +0200 Subject: [PATCH] Reland "Various test fixes for python3 support. (#11769)" --- tools/wptserve/tests/functional/base.py | 2 +- .../tests/functional/test_handlers.py | 3 -- tools/wptserve/tests/functional/test_pipes.py | 28 ++++++---------- .../wptserve/tests/functional/test_request.py | 3 +- tools/wptserve/wptserve/pipes.py | 14 +++++--- tools/wptserve/wptserve/request.py | 32 ++++++++++++++----- 6 files changed, 45 insertions(+), 37 deletions(-) diff --git a/tools/wptserve/tests/functional/base.py b/tools/wptserve/tests/functional/base.py index e49c8287c5985a..741ab0bcb74575 100644 --- a/tools/wptserve/tests/functional/base.py +++ b/tools/wptserve/tests/functional/base.py @@ -75,7 +75,7 @@ def request(self, path, query=None, method="GET", headers=None, body=None, auth= req.add_data(body) if auth is not None: - req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth)) + req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8"))) return urlopen(req) diff --git a/tools/wptserve/tests/functional/test_handlers.py b/tools/wptserve/tests/functional/test_handlers.py index 7ac410abfee2d0..759a096b799570 100644 --- a/tools/wptserve/tests/functional/test_handlers.py +++ b/tools/wptserve/tests/functional/test_handlers.py @@ -88,19 +88,16 @@ def test_range_invalid(self): self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)}) self.assertEqual(cm.exception.code, 416) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.sub.txt") expected = b"localhost localhost %i" % self.server.port assert resp.read().rstrip() == expected - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"}) expected = b"PASS" assert resp.read().rstrip() == expected - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.sub.txt", query="test=PASS") expected = b"PASS" diff --git a/tools/wptserve/tests/functional/test_pipes.py b/tools/wptserve/tests/functional/test_pipes.py index 7aa73353386624..7739af5e26e17c 100644 --- a/tools/wptserve/tests/functional/test_pipes.py +++ b/tools/wptserve/tests/functional/test_pipes.py @@ -57,40 +57,35 @@ def test_no_lower(self): self.assertEqual(resp.read(), expected[:10]) class TestSub(TestUsingServer): - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.txt", query="pipe=sub") - expected = "localhost localhost %i" % self.server.port + expected = b"localhost localhost %i" % self.server.port self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_file_hash(self): resp = self.request("/sub_file_hash.sub.txt") - expected = """ + expected = b""" md5: JmI1W8fMHfSfCarYOSxJcw== sha1: nqpWqEw4IW8NjD6R375gtrQvtTo= sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ== sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw= sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E -sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz -JwGFonfXwg==""" +sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg==""" self.assertEqual(resp.read().rstrip(), expected.strip()) def test_sub_file_hash_unrecognized(self): with self.assertRaises(urllib.error.HTTPError): self.request("/sub_file_hash_unrecognized.sub.txt") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"}) - expected = "PASS" + expected = b"PASS" self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_location(self): resp = self.request("/sub_location.sub.txt?query_string") expected = """ @@ -101,30 +96,27 @@ def test_sub_location(self): port: {0} query: ?query_string scheme: http -server: http://localhost:{0}""".format(self.server.port) +server: http://localhost:{0}""".format(self.server.port).encode("ascii") self.assertEqual(resp.read().rstrip(), expected.strip()) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub") - expected = "PASS" + expected = b"PASS" self.assertEqual(resp.read().rstrip(), expected) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_url_base(self): resp = self.request("/sub_url_base.sub.txt") - self.assertEqual(resp.read().rstrip(), "Before / After") + self.assertEqual(resp.read().rstrip(), b"Before / After") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_uuid(self): resp = self.request("/sub_uuid.sub.txt") - self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After") + self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_var(self): resp = self.request("/sub_var.sub.txt") port = self.server.port - expected = "localhost %s A %s B localhost C" % (port, port) + print(port, type(port)) + expected = b"localhost %d A %d B localhost C" % (port, port) self.assertEqual(resp.read().rstrip(), expected) class TestTrickle(TestUsingServer): diff --git a/tools/wptserve/tests/functional/test_request.py b/tools/wptserve/tests/functional/test_request.py index 096c0535b9ae79..faea1da742fefc 100644 --- a/tools/wptserve/tests/functional/test_request.py +++ b/tools/wptserve/tests/functional/test_request.py @@ -117,7 +117,6 @@ def handler(request, response): class TestAuth(TestUsingServer): - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_auth(self): @wptserve.handlers.handler def handler(request, response): @@ -127,4 +126,4 @@ def handler(request, response): self.server.router.register(*route) resp = self.request(route[1], auth=("test", "PASS")) self.assertEqual(200, resp.getcode()) - self.assertEqual(["test", "PASS"], resp.read().split(" ")) + self.assertEqual([b"test", b"PASS"], resp.read().split(b" ")) diff --git a/tools/wptserve/wptserve/pipes.py b/tools/wptserve/wptserve/pipes.py index b2410d5006f052..737b14d11cff82 100644 --- a/tools/wptserve/wptserve/pipes.py +++ b/tools/wptserve/wptserve/pipes.py @@ -1,5 +1,6 @@ from cgi import escape from collections import deque +import base64 import gzip as gzip_module import hashlib import os @@ -393,7 +394,7 @@ def uuid(request): @staticmethod def file_hash(request, algorithm, path): - algorithm = algorithm.decode("ascii") + assert isinstance(algorithm, text_type) if algorithm not in SubFunctions.supported_algorithms: raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm) @@ -401,7 +402,7 @@ def file_hash(request, algorithm, path): absolute_path = os.path.join(request.doc_root, path) try: - with open(absolute_path) as f: + with open(absolute_path, "rb") as f: hash_obj.update(f.read()) except IOError: # In this context, an unhandled IOError will be interpreted by the @@ -411,7 +412,7 @@ def file_hash(request, algorithm, path): # the path to the file to be hashed is invalid. raise Exception('Cannot open file for hash computation: "%s"' % absolute_path) - return hash_obj.digest().encode('base64').strip() + return base64.b64encode(hash_obj.digest()).strip() def template(request, content, escape_type="html"): #TODO: There basically isn't any error handling here @@ -426,7 +427,6 @@ def config_replacement(match): tokens = deque(tokens) token_type, field = tokens.popleft() - field = field.decode("ascii") if token_type == "var": variable = field @@ -491,7 +491,11 @@ def config_replacement(match): #Should possibly support escaping for other contexts e.g. script #TODO: read the encoding of the response - return escape_func(text_type(value)).encode("utf-8") + if isinstance(value, binary_type): + value = value.decode("utf-8") + elif isinstance(value, int): + value = text_type(value) + return escape_func(value).encode("utf-8") template_regexp = re.compile(br"{{([^}]*)}}") new_content = template_regexp.sub(config_replacement, content) diff --git a/tools/wptserve/wptserve/request.py b/tools/wptserve/wptserve/request.py index cb575ccdba4514..233ff151c28e48 100644 --- a/tools/wptserve/wptserve/request.py +++ b/tools/wptserve/wptserve/request.py @@ -1,7 +1,7 @@ import base64 import cgi from six.moves.http_cookies import BaseCookie -from six import BytesIO +from six import BytesIO, binary_type, text_type import tempfile from six.moves.urllib.parse import parse_qsl, urlsplit @@ -318,8 +318,8 @@ def POST(self): def cookies(self): if self._cookies is None: parser = BaseCookie() - cookie_headers = self.headers.get("cookie", "") - parser.load(cookie_headers) + cookie_headers = self.headers.get("cookie", u"") + parser.load(cookie_headers.encode("utf-8")) cookies = Cookies() for key, value in parser.iteritems(): cookies[key] = CookieValue(value) @@ -355,6 +355,16 @@ def __init__(self, request_handler): super(H2Request, self).__init__(request_handler) +def maybedecode(s): + if isinstance(s, text_type): + return s + + if isinstance(s, binary_type): + return s.decode("ascii") + + raise TypeError("Unexpected value in RequestHeaders: %r" % s) + + class RequestHeaders(dict): """Dictionary-like API for accessing request headers.""" def __init__(self, items): @@ -369,15 +379,17 @@ def __init__(self, items): for value in values: # getallmatchingheaders returns raw header lines, so # split to get name, value - multiples.append(value.split(':', 1)[1].strip()) - dict.__setitem__(self, key, multiples) + multiples.append(maybedecode(value).split(':', 1)[1].strip()) + headers = multiples else: - dict.__setitem__(self, key, [items[header]]) + headers = [maybedecode(items[header])] + dict.__setitem__(self, maybedecode(key), headers) def __getitem__(self, key): """Get all headers of a certain (case-insensitive) name. If there is more than one, the values are returned comma separated""" + key = maybedecode(key) values = dict.__getitem__(self, key.lower()) if len(values) == 1: return values[0] @@ -403,6 +415,7 @@ def get(self, key, default=None): def get_list(self, key, default=missing): """Get all the header values for a particular field name as a list""" + key = maybedecode(key) try: return dict.__getitem__(self, key.lower()) except KeyError: @@ -412,6 +425,7 @@ def get_list(self, key, default=missing): raise def __contains__(self, key): + key = maybedecode(key) return dict.__contains__(self, key.lower()) def iteritems(self): @@ -599,6 +613,7 @@ def __init__(self, headers): if "authorization" in headers: header = headers.get("authorization") + assert isinstance(header, text_type) auth_type, data = header.split(" ", 1) if auth_type in auth_schemes: self.username, self.password = auth_schemes[auth_type](data) @@ -606,5 +621,6 @@ def __init__(self, headers): raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) def decode_basic(self, data): - decoded_data = base64.decodestring(data) - return decoded_data.split(":", 1) + assert isinstance(data, text_type) + decoded_data = base64.decodestring(data.encode("utf-8")) + return decoded_data.decode("utf-8").split(":", 1)