Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#454: Don't append CRLF for the last read line #455

Merged
merged 1 commit into from
Aug 3, 2015
Merged

#454: Don't append CRLF for the last read line #455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions aiohttp/multipart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import warnings
import zlib
from urllib.parse import quote, unquote, urlencode, parse_qsl
from collections import Mapping, Sequence
from collections import deque, Mapping, Sequence

from .helpers import parse_mimetype
from .multidict import CIMultiDict
Expand Down Expand Up @@ -200,7 +200,7 @@ def __init__(self, boundary, headers, content):
length = self.headers.get(CONTENT_LENGTH, None)
self._length = int(length) if length is not None else None
self._read_bytes = 0
self._unread = []
self._unread = deque()

@asyncio.coroutine
def next(self):
Expand Down Expand Up @@ -262,7 +262,12 @@ def readline(self):
"""
if self._at_eof:
return b''
line = yield from self._content.readline()

if self._unread:
line = self._unread.popleft()
else:
line = yield from self._content.readline()

if line.startswith(self._boundary):
# the very last boundary may not come with \r\n,
# so set single rules for everyone
Expand All @@ -274,6 +279,12 @@ def readline(self):
self._at_eof = True
self._unread.append(line)
return b''
else:
next_line = yield from self._content.readline()
if next_line.startswith(self._boundary):
line = line[:-2] # strip CRLF but only once
self._unread.append(next_line)

return line

@asyncio.coroutine
Expand Down
28 changes: 14 additions & 14 deletions tests/test_multipart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,14 @@ def test_next(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello, world!\r\n--:'))
result = yield from obj.next()
self.assertEqual(b'Hello, world!\r\n', result)
self.assertEqual(b'Hello, world!', result)
self.assertTrue(obj.at_eof())

def test_next_next(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello, world!\r\n--:'))
result = yield from obj.next()
self.assertEqual(b'Hello, world!\r\n', result)
self.assertEqual(b'Hello, world!', result)
self.assertTrue(obj.at_eof())
result = yield from obj.next()
self.assertIsNone(result)
Expand All @@ -132,7 +132,7 @@ def test_read(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello, world!\r\n--:'))
result = yield from obj.read()
self.assertEqual(b'Hello, world!\r\n', result)
self.assertEqual(b'Hello, world!', result)
self.assertTrue(obj.at_eof())

def test_read_chunk_at_eof(self):
Expand All @@ -153,15 +153,15 @@ def test_read_does_reads_boundary(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, stream)
result = yield from obj.read()
self.assertEqual(b'Hello, world!\r\n', result)
self.assertEqual(b'Hello, world!', result)
self.assertEqual(b'', (yield from stream.read()))
self.assertEqual([b'--:'], obj._unread)
self.assertEqual([b'--:'], list(obj._unread))

def test_multiread(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--'))
result = yield from obj.read()
self.assertEqual(b'Hello,\r\n', result)
self.assertEqual(b'Hello,', result)
result = yield from obj.read()
self.assertEqual(b'', result)
self.assertTrue(obj.at_eof())
Expand All @@ -170,7 +170,7 @@ def test_read_multiline(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--'))
result = yield from obj.read()
self.assertEqual(b'Hello\n,\r\nworld!\r\n', result)
self.assertEqual(b'Hello\n,\r\nworld!', result)
result = yield from obj.read()
self.assertEqual(b'', result)
self.assertTrue(obj.at_eof())
Expand Down Expand Up @@ -207,7 +207,7 @@ def test_read_with_content_encoding_identity(self):
self.boundary, {CONTENT_ENCODING: 'identity'},
Stream(thing + b'--:--'))
result = yield from obj.read(decode=True)
self.assertEqual(thing, result)
self.assertEqual(thing[:-2], result)

def test_read_with_content_encoding_unknown(self):
obj = aiohttp.multipart.BodyPartReader(
Expand All @@ -230,7 +230,7 @@ def test_read_with_content_transfer_encoding_quoted_printable(self):
b' =D0=BC=D0=B8=D1=80!\r\n--:--'))
result = yield from obj.read(decode=True)
self.assertEqual(b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,'
b' \xd0\xbc\xd0\xb8\xd1\x80!\r\n', result)
b' \xd0\xbc\xd0\xb8\xd1\x80!', result)

def test_read_with_content_transfer_encoding_unknown(self):
obj = aiohttp.multipart.BodyPartReader(
Expand All @@ -243,21 +243,21 @@ def test_read_text(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {}, Stream(b'Hello, world!\r\n--:--'))
result = yield from obj.text()
self.assertEqual('Hello, world!\r\n', result)
self.assertEqual('Hello, world!', result)

def test_read_text_encoding(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {},
Stream('Привет, Мир!\r\n--:--'.encode('cp1251')))
result = yield from obj.text(encoding='cp1251')
self.assertEqual('Привет, Мир!\r\n', result)
self.assertEqual('Привет, Мир!', result)

def test_read_text_guess_encoding(self):
obj = aiohttp.multipart.BodyPartReader(
self.boundary, {CONTENT_TYPE: 'text/plain;charset=cp1251'},
Stream('Привет, Мир!\r\n--:--'.encode('cp1251')))
result = yield from obj.text()
self.assertEqual('Привет, Мир!\r\n', result)
self.assertEqual('Привет, Мир!', result)

def test_read_text_compressed(self):
obj = aiohttp.multipart.BodyPartReader(
Expand Down Expand Up @@ -352,7 +352,7 @@ def test_release(self):
yield from obj.release()
self.assertTrue(obj.at_eof())
self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read())
self.assertEqual([b'--:\r\n'], obj._unread)
self.assertEqual([b'--:\r\n'], list(obj._unread))

def test_release_respects_content_length(self):
obj = aiohttp.multipart.BodyPartReader(
Expand All @@ -369,7 +369,7 @@ def test_release_release(self):
yield from obj.release()
yield from obj.release()
self.assertEqual(b'\r\nworld!\r\n--:--', stream.content.read())
self.assertEqual([b'--:\r\n'], obj._unread)
self.assertEqual([b'--:\r\n'], list(obj._unread))

def test_filename(self):
part = aiohttp.multipart.BodyPartReader(
Expand Down