From 9bf34b7a0df0f761bc095c7fae0f8f6e6a0186f9 Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Mon, 3 Aug 2015 00:19:40 +0300 Subject: [PATCH] #454: Don't append CRLF for the last read line The RFC http://www.w3.org/Protocols/rfc1341/7_2_Multipart.html says: "NOTE: The CRLF preceding the encapsulation line is considered part of the boundary so that it is possible to have a part that does not end with a CRLF (line break). Body parts that must be considered to end with line breaks, therefore, should have two CRLFs preceding the encapsulation line, the first of which is part of the preceding body part, and the second of which is part of the encapsulation boundary." --- aiohttp/multipart.py | 17 ++++++++++++++--- tests/test_multipart.py | 28 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 296c6790506..e3ea335bf48 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -10,7 +10,7 @@ import warnings import zlib from urllib.parse import quote, unquote, urlencode, parse_qsl -from collections import Mapping, Sequence +from collections import deque, Mapping, Sequence from .helpers import parse_mimetype from .multidict import CIMultiDict @@ -200,7 +200,7 @@ def __init__(self, boundary, headers, content): length = self.headers.get(CONTENT_LENGTH, None) self._length = int(length) if length is not None else None self._read_bytes = 0 - self._unread = [] + self._unread = deque() @asyncio.coroutine def next(self): @@ -262,7 +262,12 @@ def readline(self): """ if self._at_eof: return b'' - line = yield from self._content.readline() + + if self._unread: + line = self._unread.popleft() + else: + line = yield from self._content.readline() + if line.startswith(self._boundary): # the very last boundary may not come with \r\n, # so set single rules for everyone @@ -274,6 +279,12 @@ def readline(self): self._at_eof = True self._unread.append(line) return b'' + else: + next_line = yield from self._content.readline() + if next_line.startswith(self._boundary): + line = line[:-2] # strip CRLF but only once + self._unread.append(next_line) + return line @asyncio.coroutine diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 2c40a6e0aa4..2d0986d71b3 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -116,14 +116,14 @@ def test_next(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello, world!\r\n--:')) result = yield from obj.next() - self.assertEqual(b'Hello, world!\r\n', result) + self.assertEqual(b'Hello, world!', result) self.assertTrue(obj.at_eof()) def test_next_next(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello, world!\r\n--:')) result = yield from obj.next() - self.assertEqual(b'Hello, world!\r\n', result) + self.assertEqual(b'Hello, world!', result) self.assertTrue(obj.at_eof()) result = yield from obj.next() self.assertIsNone(result) @@ -132,7 +132,7 @@ def test_read(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello, world!\r\n--:')) result = yield from obj.read() - self.assertEqual(b'Hello, world!\r\n', result) + self.assertEqual(b'Hello, world!', result) self.assertTrue(obj.at_eof()) def test_read_chunk_at_eof(self): @@ -153,15 +153,15 @@ def test_read_does_reads_boundary(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, stream) result = yield from obj.read() - self.assertEqual(b'Hello, world!\r\n', result) + self.assertEqual(b'Hello, world!', result) self.assertEqual(b'', (yield from stream.read())) - self.assertEqual([b'--:'], obj._unread) + self.assertEqual([b'--:'], list(obj._unread)) def test_multiread(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--')) result = yield from obj.read() - self.assertEqual(b'Hello,\r\n', result) + self.assertEqual(b'Hello,', result) result = yield from obj.read() self.assertEqual(b'', result) self.assertTrue(obj.at_eof()) @@ -170,7 +170,7 @@ def test_read_multiline(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--')) result = yield from obj.read() - self.assertEqual(b'Hello\n,\r\nworld!\r\n', result) + self.assertEqual(b'Hello\n,\r\nworld!', result) result = yield from obj.read() self.assertEqual(b'', result) self.assertTrue(obj.at_eof()) @@ -207,7 +207,7 @@ def test_read_with_content_encoding_identity(self): self.boundary, {CONTENT_ENCODING: 'identity'}, Stream(thing + b'--:--')) result = yield from obj.read(decode=True) - self.assertEqual(thing, result) + self.assertEqual(thing[:-2], result) def test_read_with_content_encoding_unknown(self): obj = aiohttp.multipart.BodyPartReader( @@ -230,7 +230,7 @@ def test_read_with_content_transfer_encoding_quoted_printable(self): b' =D0=BC=D0=B8=D1=80!\r\n--:--')) result = yield from obj.read(decode=True) self.assertEqual(b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,' - b' \xd0\xbc\xd0\xb8\xd1\x80!\r\n', result) + b' \xd0\xbc\xd0\xb8\xd1\x80!', result) def test_read_with_content_transfer_encoding_unknown(self): obj = aiohttp.multipart.BodyPartReader( @@ -243,21 +243,21 @@ def test_read_text(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream(b'Hello, world!\r\n--:--')) result = yield from obj.text() - self.assertEqual('Hello, world!\r\n', result) + self.assertEqual('Hello, world!', result) def test_read_text_encoding(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) result = yield from obj.text(encoding='cp1251') - self.assertEqual('Привет, Мир!\r\n', result) + self.assertEqual('Привет, Мир!', result) def test_read_text_guess_encoding(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {CONTENT_TYPE: 'text/plain;charset=cp1251'}, Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) result = yield from obj.text() - self.assertEqual('Привет, Мир!\r\n', result) + self.assertEqual('Привет, Мир!', result) def test_read_text_compressed(self): obj = aiohttp.multipart.BodyPartReader( @@ -352,7 +352,7 @@ def test_release(self): yield from obj.release() self.assertTrue(obj.at_eof()) self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read()) - self.assertEqual([b'--:\r\n'], obj._unread) + self.assertEqual([b'--:\r\n'], list(obj._unread)) def test_release_respects_content_length(self): obj = aiohttp.multipart.BodyPartReader( @@ -369,7 +369,7 @@ def test_release_release(self): yield from obj.release() yield from obj.release() self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read()) - self.assertEqual([b'--:\r\n'], obj._unread) + self.assertEqual([b'--:\r\n'], list(obj._unread)) def test_filename(self): part = aiohttp.multipart.BodyPartReader(