From a78092583fa021f65421dfd5f7045262dc218453 Mon Sep 17 00:00:00 2001 From: Arthur Darcet Date: Mon, 9 Jan 2017 15:22:14 +0100 Subject: [PATCH 1/3] do not use readline when reading the content of a part in the multipart reader --- aiohttp/multipart.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 26740da1822..be8cb2e0278 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -252,12 +252,8 @@ def read(self, *, decode=False): if self._at_eof: return b'' data = bytearray() - if self._length is None: - while not self._at_eof: - data.extend((yield from self.readline())) - else: - while not self._at_eof: - data.extend((yield from self.read_chunk(self.chunk_size))) + while not self._at_eof: + data.extend((yield from self.read_chunk(self.chunk_size))) if decode: return self.decode(data) return data @@ -377,12 +373,8 @@ def release(self): """ if self._at_eof: return - if self._length is None: - while not self._at_eof: - yield from self.readline() - else: - while not self._at_eof: - yield from self.read_chunk(self.chunk_size) + while not self._at_eof: + yield from self.read_chunk(self.chunk_size) @asyncio.coroutine def text(self, *, encoding=None): From 0f78faaabe1176cfca2f3a899a2de3eb7824e149 Mon Sep 17 00:00:00 2001 From: Arthur Darcet Date: Mon, 9 Jan 2017 16:44:12 +0100 Subject: [PATCH 2/3] fix the tests --- tests/test_multipart.py | 11 ++++------- tests/test_py35/test_multipart_35.py | 6 ++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 2387647351b..04e29793193 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -255,14 +255,13 @@ def test_read_chunk_properly_counts_read_bytes(self): self.assertEqual(b'.' * size, result) self.assertTrue(obj.at_eof()) - def test_read_does_reads_boundary(self): + def test_read_does_not_read_boundary(self): stream = Stream(b'Hello, world!\r\n--:') obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, stream) result = yield from obj.read() self.assertEqual(b'Hello, world!', result) - self.assertEqual(b'', (yield from stream.read())) - self.assertEqual([b'--:'], list(obj._unread)) + self.assertEqual(b'--:', (yield from stream.read())) def test_multiread(self): obj = aiohttp.multipart.BodyPartReader( @@ -474,8 +473,7 @@ def test_release(self): self.boundary, {}, stream) yield from obj.release() self.assertTrue(obj.at_eof()) - self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read()) - self.assertEqual([b'--:\r\n'], list(obj._unread)) + self.assertEqual(b'--:\r\n\r\nworld!\r\n--:--', stream.content.read()) def test_release_respects_content_length(self): obj = aiohttp.multipart.BodyPartReader( @@ -491,8 +489,7 @@ def test_release_release(self): self.boundary, {}, stream) yield from obj.release() yield from obj.release() - self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read()) - self.assertEqual([b'--:\r\n'], list(obj._unread)) + self.assertEqual(b'--:\r\n\r\nworld!\r\n--:--', stream.content.read()) def test_filename(self): part = aiohttp.multipart.BodyPartReader( diff --git a/tests/test_py35/test_multipart_35.py b/tests/test_py35/test_multipart_35.py index 8a89e585af5..cb9300fd296 100644 --- a/tests/test_py35/test_multipart_35.py +++ b/tests/test_py35/test_multipart_35.py @@ -13,9 +13,15 @@ def __init__(self, content): async def read(self, size=None): return self.content.read(size) + def at_eof(self): + return self.content.tell() == len(self.content.getbuffer()) + async def readline(self): return self.content.readline() + def unread_data(self, data): + self.content = io.BytesIO(data + self.content.read()) + async def test_async_for_reader(loop): data = [{"test": "passed"}, 42, b'plain text', b'aiohttp\n'] From 03101a0751f9e97ded99539c45f0384ab1e10d13 Mon Sep 17 00:00:00 2001 From: Arthur Darcet Date: Mon, 9 Jan 2017 16:48:07 +0100 Subject: [PATCH 3/3] add regression test --- tests/test_multipart.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 04e29793193..708ad0e1560 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -13,6 +13,8 @@ from aiohttp.helpers import parse_mimetype from aiohttp.multipart import (content_disposition_filename, parse_content_disposition) +from aiohttp.streams import DEFAULT_LIMIT as stream_reader_default_limit +from aiohttp.streams import StreamReader def run_in_loop(f): @@ -498,6 +500,16 @@ def test_filename(self): None) self.assertEqual('foo.html', part.filename) + def test_reading_long_part(self): + size = 2 * stream_reader_default_limit + stream = StreamReader() + stream.feed_data(b'0' * size + b'\r\n--:--') + stream.feed_eof() + obj = aiohttp.multipart.BodyPartReader( + self.boundary, {}, stream) + data = yield from obj.read() + self.assertEqual(len(data), size) + class MultipartReaderTestCase(TestCase):