From 974daa54ef05a491890fd16787e15ee1f5115210 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 23 Jul 2021 19:27:13 -0700 Subject: [PATCH] Read smaller frames to workaround OpenSSL bug As older versions of OpenSSL (in particular 1.0.2) have limitations on the size of buffers they can work with, take small views into our larger buffer and read those in instead. This should keep the buffer sizes more manageable for OpenSSL. --- distributed/comm/tcp.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index de6cee016d8..05b03f562cf 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -193,9 +193,14 @@ async def read(self, deserializers=None): frames_nbytes = await stream.read_bytes(fmt_size) (frames_nbytes,) = struct.unpack(fmt, frames_nbytes) - frames = bytearray(frames_nbytes) - n = await stream.read_into(frames) - assert n == frames_nbytes, (n, frames_nbytes) + frames = memoryview(bytearray(frames_nbytes)) + # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) + MAX = 1_000 + for i, j in sliding_window(range(0, frame_nbytes + MAX, MAX)): + chunk = frames[i:j] + chunk_nbytes = len(chunk) + n = await stream.read_into(chunk) + assert n == chunk_nbytes, (n, chunk_nbytes) except StreamClosedError as e: self.stream = None self._closed = True