From b50c5b07e419d2b926bcddeef944185324b2474d Mon Sep 17 00:00:00 2001 From: Geoffrey Sneddon Date: Fri, 28 Sep 2018 00:43:21 +0100 Subject: [PATCH] Revert "Various test fixes for python3 support. (#11769)" This reverts commit 5bd512cdc76f762662915a2d3d9e49b021f14ff6. Fixes #13204. --- tools/wptserve/tests/functional/base.py | 2 +- .../tests/functional/test_handlers.py | 3 ++ tools/wptserve/tests/functional/test_pipes.py | 52 ++++++++++++------- .../wptserve/tests/functional/test_request.py | 3 +- tools/wptserve/wptserve/pipes.py | 19 +++---- tools/wptserve/wptserve/request.py | 32 +++--------- tools/wptserve/wptserve/response.py | 4 +- 7 files changed, 56 insertions(+), 59 deletions(-) diff --git a/tools/wptserve/tests/functional/base.py b/tools/wptserve/tests/functional/base.py index 741ab0bcb745753..e49c8287c5985aa 100644 --- a/tools/wptserve/tests/functional/base.py +++ b/tools/wptserve/tests/functional/base.py @@ -75,7 +75,7 @@ def request(self, path, query=None, method="GET", headers=None, body=None, auth= req.add_data(body) if auth is not None: - req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8"))) + req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth)) return urlopen(req) diff --git a/tools/wptserve/tests/functional/test_handlers.py b/tools/wptserve/tests/functional/test_handlers.py index 759a096b7995707..7ac410abfee2d0a 100644 --- a/tools/wptserve/tests/functional/test_handlers.py +++ b/tools/wptserve/tests/functional/test_handlers.py @@ -88,16 +88,19 @@ def test_range_invalid(self): self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)}) self.assertEqual(cm.exception.code, 416) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.sub.txt") expected = b"localhost localhost %i" % self.server.port assert resp.read().rstrip() == expected + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"}) expected = b"PASS" assert resp.read().rstrip() == expected + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.sub.txt", query="test=PASS") expected = b"PASS" diff --git a/tools/wptserve/tests/functional/test_pipes.py b/tools/wptserve/tests/functional/test_pipes.py index 7739af5e26e17c7..83b2c621641ca17 100644 --- a/tools/wptserve/tests/functional/test_pipes.py +++ b/tools/wptserve/tests/functional/test_pipes.py @@ -57,35 +57,40 @@ def test_no_lower(self): self.assertEqual(resp.read(), expected[:10]) class TestSub(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.txt", query="pipe=sub") - expected = b"localhost localhost %i" % self.server.port + expected = "localhost localhost %i" % self.server.port self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_file_hash(self): resp = self.request("/sub_file_hash.sub.txt") - expected = b""" + expected = """ md5: JmI1W8fMHfSfCarYOSxJcw== sha1: nqpWqEw4IW8NjD6R375gtrQvtTo= sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ== sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw= sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E -sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg==""" +sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz +JwGFonfXwg==""" self.assertEqual(resp.read().rstrip(), expected.strip()) def test_sub_file_hash_unrecognized(self): with self.assertRaises(urllib.error.HTTPError): self.request("/sub_file_hash_unrecognized.sub.txt") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"}) - expected = b"PASS" + expected = "PASS" self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_location(self): resp = self.request("/sub_location.sub.txt?query_string") expected = """ @@ -96,27 +101,30 @@ def test_sub_location(self): port: {0} query: ?query_string scheme: http -server: http://localhost:{0}""".format(self.server.port).encode("ascii") +server: http://localhost:{0}""".format(self.server.port) self.assertEqual(resp.read().rstrip(), expected.strip()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub") - expected = b"PASS" + expected = "PASS" self.assertEqual(resp.read().rstrip(), expected) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_url_base(self): resp = self.request("/sub_url_base.sub.txt") - self.assertEqual(resp.read().rstrip(), b"Before / After") + self.assertEqual(resp.read().rstrip(), "Before / After") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_uuid(self): resp = self.request("/sub_uuid.sub.txt") - self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After") + self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_var(self): resp = self.request("/sub_var.sub.txt") port = self.server.port - print(port, type(port)) - expected = b"localhost %d A %d B localhost C" % (port, port) + expected = "localhost %s A %s B localhost C" % (port, port) self.assertEqual(resp.read().rstrip(), expected) class TestTrickle(TestUsingServer): @@ -136,10 +144,12 @@ def test_headers(self): self.assertEqual(resp.info()["Expires"], "0") class TestPipesWithVariousHandlers(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_file_handler(self): resp = self.request("/test_string.py", query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b"PA") + self.assertEqual(resp.read(), "PA") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler(self): @wptserve.handlers.handler def handler(request, response): @@ -147,8 +157,9 @@ def handler(request, response): route = ("GET", "/test/test_pipes_1/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b"PA") + self.assertEqual(resp.read(), "PA") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): @@ -157,8 +168,9 @@ def handler(request, response): self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") # slice has not been applied to the response, because response.writer was used. - self.assertEqual(resp.read(), b"PASS") + self.assertEqual(resp.read(), "PASS") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_header_pipe_with_python_func_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): @@ -168,8 +180,9 @@ def handler(request, response): resp = self.request(route[1], query="pipe=header(X-TEST,FAIL)") # header pipe was ignored, because response.writer was used. self.assertFalse(resp.info().get("X-TEST")) - self.assertEqual(resp.read(), b"CONTENT") + self.assertEqual(resp.read(), "CONTENT") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_json_handler(self): @wptserve.handlers.json_handler def handler(request, response): @@ -177,29 +190,32 @@ def handler(request, response): route = ("GET", "/test/test_pipes_2/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b'"{') + self.assertEqual(resp.read(), '"{') + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_slice_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=slice(null,2)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) self.assertEqual("PASS", resp.info()["X-Test"]) # slice has not been applied to the response, because response.writer was used. - self.assertEqual(b"Content", resp.read()) + self.assertEqual("Content", resp.read()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_headers_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=header(X-TEST,FAIL)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) # header pipe was ignored. self.assertEqual("PASS", resp.info()["X-TEST"]) - self.assertEqual(b"Content", resp.read()) + self.assertEqual("Content", resp.read()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_trickle_with_as_is_handler(self): t0 = time.time() resp = self.request("/test.asis", query="pipe=trickle(1:d2:5:d1:r2)") t1 = time.time() - self.assertTrue(b'Content' in resp.read()) + self.assertTrue('Content' in resp.read()) self.assertGreater(6, t1-t0) if __name__ == '__main__': diff --git a/tools/wptserve/tests/functional/test_request.py b/tools/wptserve/tests/functional/test_request.py index 983539260a5215c..a9ff8e8b4eb23bf 100644 --- a/tools/wptserve/tests/functional/test_request.py +++ b/tools/wptserve/tests/functional/test_request.py @@ -115,6 +115,7 @@ def handler(request, response): class TestAuth(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_auth(self): @wptserve.handlers.handler def handler(request, response): @@ -124,4 +125,4 @@ def handler(request, response): self.server.router.register(*route) resp = self.request(route[1], auth=("test", "PASS")) self.assertEqual(200, resp.getcode()) - self.assertEqual([b"test", b"PASS"], resp.read().split(b" ")) + self.assertEqual(["test", "PASS"], resp.read().split(" ")) diff --git a/tools/wptserve/wptserve/pipes.py b/tools/wptserve/wptserve/pipes.py index 33a43e1f06b9835..a0f4a26be380848 100644 --- a/tools/wptserve/wptserve/pipes.py +++ b/tools/wptserve/wptserve/pipes.py @@ -1,6 +1,5 @@ from cgi import escape from collections import deque -import base64 import gzip as gzip_module import hashlib import os @@ -274,9 +273,8 @@ def slice(request, response, start, end=None): (spelled "null" in a query string) to indicate the end of the file. """ - content = resolve_content(response)[start:end] - response.content = content - response.headers.set("Content-Length", len(content)) + content = resolve_content(response) + response.content = content[start:end] return response @@ -394,7 +392,7 @@ def uuid(request): @staticmethod def file_hash(request, algorithm, path): - assert isinstance(algorithm, text_type) + algorithm = algorithm.decode("ascii") if algorithm not in SubFunctions.supported_algorithms: raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm) @@ -402,7 +400,7 @@ def file_hash(request, algorithm, path): absolute_path = os.path.join(request.doc_root, path) try: - with open(absolute_path, "rb") as f: + with open(absolute_path) as f: hash_obj.update(f.read()) except IOError: # In this context, an unhandled IOError will be interpreted by the @@ -412,7 +410,7 @@ def file_hash(request, algorithm, path): # the path to the file to be hashed is invalid. raise Exception('Cannot open file for hash computation: "%s"' % absolute_path) - return base64.b64encode(hash_obj.digest()).strip() + return hash_obj.digest().encode('base64').strip() def template(request, content, escape_type="html"): #TODO: There basically isn't any error handling here @@ -427,6 +425,7 @@ def config_replacement(match): tokens = deque(tokens) token_type, field = tokens.popleft() + field = field.decode("ascii") if token_type == "var": variable = field @@ -494,11 +493,7 @@ def config_replacement(match): #Should possibly support escaping for other contexts e.g. script #TODO: read the encoding of the response - if isinstance(value, binary_type): - value = value.decode("utf-8") - elif isinstance(value, int): - value = text_type(value) - return escape_func(value).encode("utf-8") + return escape_func(text_type(value)).encode("utf-8") template_regexp = re.compile(br"{{([^}]*)}}") new_content = template_regexp.sub(config_replacement, content) diff --git a/tools/wptserve/wptserve/request.py b/tools/wptserve/wptserve/request.py index 233ff151c28e482..cb575ccdba4514a 100644 --- a/tools/wptserve/wptserve/request.py +++ b/tools/wptserve/wptserve/request.py @@ -1,7 +1,7 @@ import base64 import cgi from six.moves.http_cookies import BaseCookie -from six import BytesIO, binary_type, text_type +from six import BytesIO import tempfile from six.moves.urllib.parse import parse_qsl, urlsplit @@ -318,8 +318,8 @@ def POST(self): def cookies(self): if self._cookies is None: parser = BaseCookie() - cookie_headers = self.headers.get("cookie", u"") - parser.load(cookie_headers.encode("utf-8")) + cookie_headers = self.headers.get("cookie", "") + parser.load(cookie_headers) cookies = Cookies() for key, value in parser.iteritems(): cookies[key] = CookieValue(value) @@ -355,16 +355,6 @@ def __init__(self, request_handler): super(H2Request, self).__init__(request_handler) -def maybedecode(s): - if isinstance(s, text_type): - return s - - if isinstance(s, binary_type): - return s.decode("ascii") - - raise TypeError("Unexpected value in RequestHeaders: %r" % s) - - class RequestHeaders(dict): """Dictionary-like API for accessing request headers.""" def __init__(self, items): @@ -379,17 +369,15 @@ def __init__(self, items): for value in values: # getallmatchingheaders returns raw header lines, so # split to get name, value - multiples.append(maybedecode(value).split(':', 1)[1].strip()) - headers = multiples + multiples.append(value.split(':', 1)[1].strip()) + dict.__setitem__(self, key, multiples) else: - headers = [maybedecode(items[header])] - dict.__setitem__(self, maybedecode(key), headers) + dict.__setitem__(self, key, [items[header]]) def __getitem__(self, key): """Get all headers of a certain (case-insensitive) name. If there is more than one, the values are returned comma separated""" - key = maybedecode(key) values = dict.__getitem__(self, key.lower()) if len(values) == 1: return values[0] @@ -415,7 +403,6 @@ def get(self, key, default=None): def get_list(self, key, default=missing): """Get all the header values for a particular field name as a list""" - key = maybedecode(key) try: return dict.__getitem__(self, key.lower()) except KeyError: @@ -425,7 +412,6 @@ def get_list(self, key, default=missing): raise def __contains__(self, key): - key = maybedecode(key) return dict.__contains__(self, key.lower()) def iteritems(self): @@ -613,7 +599,6 @@ def __init__(self, headers): if "authorization" in headers: header = headers.get("authorization") - assert isinstance(header, text_type) auth_type, data = header.split(" ", 1) if auth_type in auth_schemes: self.username, self.password = auth_schemes[auth_type](data) @@ -621,6 +606,5 @@ def __init__(self, headers): raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) def decode_basic(self, data): - assert isinstance(data, text_type) - decoded_data = base64.decodestring(data.encode("utf-8")) - return decoded_data.decode("utf-8").split(":", 1) + decoded_data = base64.decodestring(data) + return decoded_data.split(":", 1) diff --git a/tools/wptserve/wptserve/response.py b/tools/wptserve/wptserve/response.py index 483265bf0846660..44299cc994ef43e 100644 --- a/tools/wptserve/wptserve/response.py +++ b/tools/wptserve/wptserve/response.py @@ -183,10 +183,8 @@ def iter_content(self, read_file=False): True, the entire content of the file will be returned as a string facilitating non-streaming operations like template substitution. """ - if isinstance(self.content, binary_type): + if isinstance(self.content, (binary_type, text_type)): yield self.content - elif isinstance(self.content, text_type): - yield self.content.encode("utf-8") elif hasattr(self.content, "read"): if read_file: yield self.content.read()