From cdc74bf5cdc04e9190ffff769cf740d9c9afbb32 Mon Sep 17 00:00:00 2001 From: Brian Muller Date: Mon, 16 Jan 2017 10:10:10 -0500 Subject: [PATCH 1/4] accept an errors parameter for text encoding ClientResponses --- aiohttp/client_reqrep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 6df7a20474d..f52d49d5e02 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -741,7 +741,7 @@ def _get_encoding(self): return encoding @asyncio.coroutine - def text(self, encoding=None): + def text(self, encoding=None, errors='strict'): """Read response payload and decode.""" if self._content is None: yield from self.read() @@ -749,7 +749,7 @@ def text(self, encoding=None): if encoding is None: encoding = self._get_encoding() - return self._content.decode(encoding) + return self._content.decode(encoding, errors) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): From 450e0fa896b2c6a7f2079b322640af94c20f291b Mon Sep 17 00:00:00 2001 From: Brian Muller Date: Mon, 16 Jan 2017 10:17:36 -0500 Subject: [PATCH 2/4] added PR # to CHANGES.rst file --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 739d4e029e3..79ca57d62ed 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,8 @@ CHANGES - Fix typo in FAQ section "How to programmatically close websocket server-side?". +- Allow users to specify what should happen to decoding errors when calling a responses `text()` method + 1.2.0 (2016-12-17) ------------------ From 9c7f964ce8c83e5ecd549f4d9b58302371db3afa Mon Sep 17 00:00:00 2001 From: Brian Muller Date: Sat, 21 Jan 2017 16:38:39 -0500 Subject: [PATCH 3/4] add test for bad encoding alternative handling --- aiohttp/client_reqrep.py | 2 +- tests/test_client_response.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index f52d49d5e02..930dcc02b99 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -749,7 +749,7 @@ def text(self, encoding=None, errors='strict'): if encoding is None: encoding = self._get_encoding() - return self._content.decode(encoding, errors) + return self._content.decode(encoding, errors=errors) @asyncio.coroutine def json(self, *, encoding=None, loads=json.loads): diff --git a/tests/test_client_response.py b/tests/test_client_response.py index 0b33b2a4be3..529dcec38bc 100644 --- a/tests/test_client_response.py +++ b/tests/test_client_response.py @@ -139,6 +139,29 @@ def side_effect(*args, **kwargs): assert response._connection is None +@asyncio.coroutine +def test_text_bad_encoding(loop): + response = ClientResponse('get', URL('http://def-cl-resp.org')) + response._post_init(loop) + + def side_effect(*args, **kwargs): + fut = helpers.create_future(loop) + fut.set_result('{"тестkey": "пройденvalue"}'.encode('cp1251')) + return fut + + # lie about the encoding + response.headers = { + 'Content-Type': 'application/json;charset=utf-8'} + content = response.content = mock.Mock() + content.read.side_effect = side_effect + with pytest.raises(UnicodeDecodeError): + yield from response.text() + # only the valid utf-8 characters will be returned + res = yield from response.text(errors='ignore') + assert res == '{"key": "value"}' + assert response._connection is None + + @asyncio.coroutine def test_text_custom_encoding(loop): response = ClientResponse('get', URL('http://def-cl-resp.org')) From be61176194a507913f3789147e4673973733341e Mon Sep 17 00:00:00 2001 From: Brian Muller Date: Sat, 21 Jan 2017 16:47:08 -0500 Subject: [PATCH 4/4] updated CHANGES file w/ PR number --- CHANGES.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 79ca57d62ed..1dccc056ae4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,8 +12,7 @@ CHANGES - Fix typo in FAQ section "How to programmatically close websocket server-side?". -- Allow users to specify what should happen to decoding errors when calling a responses `text()` method - +- Allow users to specify what should happen to decoding errors when calling a responses `text()` method #1542 1.2.0 (2016-12-17) ------------------