From ecdf15e208a4736b6c111936079c61bfec9fd37a Mon Sep 17 00:00:00 2001 From: ptoman-pa <95256508+ptoman-pa@users.noreply.github.com> Date: Tue, 4 Jan 2022 09:28:57 -0800 Subject: [PATCH] Fixes large payload runtime exception in Datastore (issue 1633) (#2181) * Fixes runtime exception when feature values are larger than 1500 bytes in Datastore. Datastore indexes values as well as keys so large payloads are disallowed. This change clarifies that values should not be indexed. It avoids google.api_core.exceptions.InvalidArgument: 400 The value of property _ is longer than 1500 bytes. Signed-off-by: Pamela Toman * Linted Signed-off-by: Pamela Toman --- .../feast/infra/online_stores/datastore.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index 0442eda122..f8964129cd 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -196,18 +196,18 @@ def _write_minibatch( key=key, exclude_from_indexes=("created_ts", "event_ts", "values") ) - entity.update( - dict( - key=entity_key.SerializeToString(), - values={k: v.SerializeToString() for k, v in features.items()}, - event_ts=utils.make_tzaware(timestamp), - created_ts=( - utils.make_tzaware(created_ts) - if created_ts is not None - else None - ), - ) + content_entity = datastore.Entity( + exclude_from_indexes=tuple(features.keys()) ) + for k, v in features.items(): + content_entity[k] = v.SerializeToString() + entity["key"] = entity_key.SerializeToString() + entity["values"] = content_entity + entity["event_ts"] = utils.make_tzaware(timestamp) + entity["created_ts"] = ( + utils.make_tzaware(created_ts) if created_ts is not None else None + ) + entities.append(entity) with client.transaction(): client.put_multi(entities)