Merge pull request #1535 from arthurdarcet/multipart-long-line

do not use readline when reading the content of a part in the multipart reader

aiohttp/multipart.py

@@ -252,12 +252,8 @@ def read(self, *, decode=False):
         if self._at_eof:
             return b''
         data = bytearray()
-        if self._length is None:
-            while not self._at_eof:
-                data.extend((yield from self.readline()))
-        else:
-            while not self._at_eof:
-                data.extend((yield from self.read_chunk(self.chunk_size)))
+        while not self._at_eof:
+            data.extend((yield from self.read_chunk(self.chunk_size)))
         if decode:
             return self.decode(data)
         return data
@@ -377,12 +373,8 @@ def release(self):
         """
         if self._at_eof:
             return
-        if self._length is None:
-            while not self._at_eof:
-                yield from self.readline()
-        else:
-            while not self._at_eof:
-                yield from self.read_chunk(self.chunk_size)
+        while not self._at_eof:
+            yield from self.read_chunk(self.chunk_size)
 
     @asyncio.coroutine
     def text(self, *, encoding=None):

tests/test_multipart.py

@@ -13,6 +13,8 @@
 from aiohttp.helpers import parse_mimetype
 from aiohttp.multipart import (content_disposition_filename,
                                parse_content_disposition)
+from aiohttp.streams import DEFAULT_LIMIT as stream_reader_default_limit
+from aiohttp.streams import StreamReader
 
 
 def run_in_loop(f):
@@ -255,14 +257,13 @@ def test_read_chunk_properly_counts_read_bytes(self):
         self.assertEqual(b'.' * size, result)
         self.assertTrue(obj.at_eof())
 
-    def test_read_does_reads_boundary(self):
+    def test_read_does_not_read_boundary(self):
         stream = Stream(b'Hello, world!\r\n--:')
         obj = aiohttp.multipart.BodyPartReader(
             self.boundary, {}, stream)
         result = yield from obj.read()
         self.assertEqual(b'Hello, world!', result)
-        self.assertEqual(b'', (yield from stream.read()))
-        self.assertEqual([b'--:'], list(obj._unread))
+        self.assertEqual(b'--:', (yield from stream.read()))
 
     def test_multiread(self):
         obj = aiohttp.multipart.BodyPartReader(
@@ -474,8 +475,7 @@ def test_release(self):
             self.boundary, {}, stream)
         yield from obj.release()
         self.assertTrue(obj.at_eof())
-        self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read())
-        self.assertEqual([b'--:\r\n'], list(obj._unread))
+        self.assertEqual(b'--:\r\n\r\nworld!\r\n--:--', stream.content.read())
 
     def test_release_respects_content_length(self):
         obj = aiohttp.multipart.BodyPartReader(
@@ -491,8 +491,7 @@ def test_release_release(self):
             self.boundary, {}, stream)
         yield from obj.release()
         yield from obj.release()
-        self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read())
-        self.assertEqual([b'--:\r\n'], list(obj._unread))
+        self.assertEqual(b'--:\r\n\r\nworld!\r\n--:--', stream.content.read())
 
     def test_filename(self):
         part = aiohttp.multipart.BodyPartReader(
@@ -501,6 +500,16 @@ def test_filename(self):
             None)
         self.assertEqual('foo.html', part.filename)
 
+    def test_reading_long_part(self):
+        size = 2 * stream_reader_default_limit
+        stream = StreamReader()
+        stream.feed_data(b'0' * size + b'\r\n--:--')
+        stream.feed_eof()
+        obj = aiohttp.multipart.BodyPartReader(
+            self.boundary, {}, stream)
+        data = yield from obj.read()
+        self.assertEqual(len(data), size)
+
 
 class MultipartReaderTestCase(TestCase):
 

tests/test_py35/test_multipart_35.py

@@ -13,9 +13,15 @@ def __init__(self, content):
     async def read(self, size=None):
         return self.content.read(size)
 
+    def at_eof(self):
+        return self.content.tell() == len(self.content.getbuffer())
+
     async def readline(self):
         return self.content.readline()
 
+    def unread_data(self, data):
+        self.content = io.BytesIO(data + self.content.read())
+
 
 async def test_async_for_reader(loop):
     data = [{"test": "passed"}, 42, b'plain text', b'aiohttp\n']