Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve client performance when there are no auto headers to skip #10049

Merged
merged 8 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/10049.misc.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Improved performance of making requests when there are no auto headers to skip -- by :user:`bdraco`.
37 changes: 22 additions & 15 deletions aiohttp/client_reqrep.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@
__writer: Optional["asyncio.Task[None]"] = None # async task for streaming data
_continue = None # waiter future for '100 Continue' response

_skip_auto_headers: Optional["CIMultiDict[None]"] = None

# N.B.
# Adding __del__ method with self._writer closing doesn't make sense
# because _writer is instance method, thus it keeps a reference to self.
Expand Down Expand Up @@ -293,6 +295,10 @@
def __reset_writer(self, _: object = None) -> None:
self.__writer = None

@property
def skip_auto_headers(self) -> CIMultiDict[None]:
return self._skip_auto_headers or CIMultiDict()

Check warning on line 300 in aiohttp/client_reqrep.py

View check run for this annotation

Codecov / codecov/patch

aiohttp/client_reqrep.py#L300

Added line #L300 was not covered by tests
bdraco marked this conversation as resolved.
Show resolved Hide resolved
bdraco marked this conversation as resolved.
Show resolved Hide resolved

@property
def _writer(self) -> Optional["asyncio.Task[None]"]:
return self.__writer
Expand Down Expand Up @@ -404,20 +410,19 @@

def update_auto_headers(self, skip_auto_headers: Optional[Iterable[str]]) -> None:
if skip_auto_headers is not None:
self.skip_auto_headers = CIMultiDict(
self._skip_auto_headers = CIMultiDict(
(hdr, None) for hdr in sorted(skip_auto_headers)
)
used_headers = self.headers.copy()
used_headers.extend(self.skip_auto_headers) # type: ignore[arg-type]
used_headers.extend(self._skip_auto_headers) # type: ignore[arg-type]
else:
# Fast path when there are no headers to skip
# which is the most common case.
self.skip_auto_headers = CIMultiDict()
used_headers = self.headers

for hdr, val in self.DEFAULT_HEADERS.items():
if hdr not in used_headers:
self.headers.add(hdr, val)
self.headers[hdr] = val
bdraco marked this conversation as resolved.
Show resolved Hide resolved

if hdrs.USER_AGENT not in used_headers:
self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE
Expand Down Expand Up @@ -522,21 +527,20 @@
self.body = body

# enable chunked encoding if needed
if not self.chunked:
if hdrs.CONTENT_LENGTH not in self.headers:
size = body.size
if size is None:
self.chunked = True
else:
if hdrs.CONTENT_LENGTH not in self.headers:
bdraco marked this conversation as resolved.
Show resolved Hide resolved
self.headers[hdrs.CONTENT_LENGTH] = str(size)
if not self.chunked and hdrs.CONTENT_LENGTH not in self.headers:
if (size := body.size) is not None:
self.headers[hdrs.CONTENT_LENGTH] = str(size)
else:
self.chunked = True

# copy payload headers
assert body.headers
headers = self.headers
skip_headers = self._skip_auto_headers
for key, value in body.headers.items():
if key in self.headers or key in self.skip_auto_headers:
if key in headers or (skip_headers is not None and key in skip_headers):
continue
self.headers[key] = value
headers[key] = value

def update_expect_continue(self, expect: bool = False) -> None:
if expect:
Expand Down Expand Up @@ -664,7 +668,10 @@
# set default content-type
if (
self.method in self.POST_METHODS
and hdrs.CONTENT_TYPE not in self.skip_auto_headers
and (
self._skip_auto_headers is None
or hdrs.CONTENT_TYPE not in self._skip_auto_headers
)
and hdrs.CONTENT_TYPE not in self.headers
):
self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream"
Expand Down
Loading