Merge pull request #1542 from bmuller/text-encoding-error-handling

Accept an errors parameter for text decoding ClientResponses

CHANGES.rst

@@ -14,6 +14,7 @@ CHANGES
 
 - Fix typo in FAQ section "How to programmatically close websocket server-side?"
 
+- Allow users to specify what should happen to decoding errors when calling a responses `text()` method #1542
 
 1.2.0 (2016-12-17)
 ------------------

aiohttp/client_reqrep.py

@@ -741,15 +741,15 @@ def _get_encoding(self):
         return encoding
 
     @asyncio.coroutine
-    def text(self, encoding=None):
+    def text(self, encoding=None, errors='strict'):
         """Read response payload and decode."""
         if self._content is None:
             yield from self.read()
 
         if encoding is None:
             encoding = self._get_encoding()
 
-        return self._content.decode(encoding)
+        return self._content.decode(encoding, errors=errors)
 
     @asyncio.coroutine
     def json(self, *, encoding=None, loads=json.loads):

tests/test_client_response.py

@@ -139,6 +139,29 @@ def side_effect(*args, **kwargs):
     assert response._connection is None
 
 
+@asyncio.coroutine
+def test_text_bad_encoding(loop):
+    response = ClientResponse('get', URL('http://def-cl-resp.org'))
+    response._post_init(loop)
+
+    def side_effect(*args, **kwargs):
+        fut = helpers.create_future(loop)
+        fut.set_result('{"тестkey": "пройденvalue"}'.encode('cp1251'))
+        return fut
+
+    # lie about the encoding
+    response.headers = {
+        'Content-Type': 'application/json;charset=utf-8'}
+    content = response.content = mock.Mock()
+    content.read.side_effect = side_effect
+    with pytest.raises(UnicodeDecodeError):
+        yield from response.text()
+    # only the valid utf-8 characters will be returned
+    res = yield from response.text(errors='ignore')
+    assert res == '{"key": "value"}'
+    assert response._connection is None
+
+
 @asyncio.coroutine
 def test_text_custom_encoding(loop):
     response = ClientResponse('get', URL('http://def-cl-resp.org'))