From 1869b1899f690b7556db0a463bc639322ede548e Mon Sep 17 00:00:00 2001 From: Gabe Joseph Date: Fri, 23 Jul 2021 12:23:55 -0800 Subject: [PATCH] make tests pass This is a hack because in real use, I am pretty sure that `frames` will be either all memoryviews or all not, since they'll either be coming from a comm or from a bytestring. But in tests where we call `loads` directly, this may not be the case. --- distributed/protocol/core.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py index b20f272d11..be8bf9abe7 100644 --- a/distributed/protocol/core.py +++ b/distributed/protocol/core.py @@ -100,12 +100,23 @@ def _decode_default(obj): if deserialize: if "compression" in sub_header: sub_frames = decompress(sub_header, sub_frames) + + # HACK: check for memoryviews in preceding frames that share an underlying + # buffer with these sub-frames, to figure out what offset in the underlying + # buffer the sub-frames start at. + memoryview_offset = 0 + if sub_frames and isinstance(sub_frames[0], memoryview): + obj = sub_frames[0].obj + for f in reversed(frames[:offset]): + if not (isinstance(f, memoryview) and f.obj is obj): + break + memoryview_offset += len(f) + return merge_and_deserialize( sub_header, sub_frames, deserializers=deserializers, - memoryview_offset=sum(len(f) for f in frames[:offset]), - # ^ TODO: what if not all frames are memoryviews? + memoryview_offset=memoryview_offset, ) else: return Serialized(sub_header, sub_frames)