From f6c8ada75ddc4e988ba0e4ad9b1c19c2505046d5 Mon Sep 17 00:00:00 2001 From: Martin Nowak Date: Fri, 5 Apr 2019 09:38:16 +0200 Subject: [PATCH 1/2] remove gRPC size limit for bigquery storage - currently limited to 4MiB - compare to https://github.com/googleapis/google-cloud-python/pull/5594/commits/f636c8d7756a470719d114e8d77dd169e39906f3 which does the same for the monitoring API --- .../gapic/transports/big_query_storage_grpc_transport.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py b/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py index 01cd7193b442..0844039a7648 100644 --- a/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py +++ b/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py @@ -91,7 +91,11 @@ def create_channel( grpc.Channel: A gRPC channel object. """ return google.api_core.grpc_helpers.create_channel( # pragma: no cover - address, credentials=credentials, scopes=cls._OAUTH_SCOPES + address, credentials=credentials, scopes=cls._OAUTH_SCOPES, + options={ + 'grpc.max_send_message_length': -1, + 'grpc.max_receive_message_length': -1, + }.items() ) @property From a4f9ea31083af7101264790ca663f01a332c9090 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 8 Apr 2019 14:18:45 -0700 Subject: [PATCH 2/2] Add message lenght override to synth config. --- .../transports/big_query_storage_grpc_transport.py | 10 ++++++---- bigquery_storage/synth.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py b/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py index 0844039a7648..73ac2d16d5b9 100644 --- a/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py +++ b/bigquery_storage/google/cloud/bigquery_storage_v1beta1/gapic/transports/big_query_storage_grpc_transport.py @@ -91,11 +91,13 @@ def create_channel( grpc.Channel: A gRPC channel object. """ return google.api_core.grpc_helpers.create_channel( # pragma: no cover - address, credentials=credentials, scopes=cls._OAUTH_SCOPES, + address, + credentials=credentials, + scopes=cls._OAUTH_SCOPES, options={ - 'grpc.max_send_message_length': -1, - 'grpc.max_receive_message_length': -1, - }.items() + "grpc.max_send_message_length": -1, + "grpc.max_receive_message_length": -1, + }.items(), ) @property diff --git a/bigquery_storage/synth.py b/bigquery_storage/synth.py index 252e4bc53b06..fb8da2c7005e 100644 --- a/bigquery_storage/synth.py +++ b/bigquery_storage/synth.py @@ -75,6 +75,17 @@ "big_query_storage_client.BigQueryStorageClient", ) +# The grpc transport channel shouldn't limit the size of a grpc message at the +# default 4mb. +s.replace("google/cloud/bigquery_storage_v1beta1/gapic/transports/*_grpc_transport.py", + "return google.api_core.grpc_helpers.create_channel\(\n(\s+)address,\n" + "\s+credentials=.*,\n\s+scopes=.*,\n", + "\g<0>\g<1>options={\n" + "\g<1> 'grpc.max_send_message_length': -1,\n" + "\g<1> 'grpc.max_receive_message_length': -1,\n" + "\g<1>}.items(),\n", +) + # START: Ignore lint and coverage s.replace( ["google/cloud/bigquery_storage_v1beta1/gapic/big_query_storage_client.py"],