Skip to content

Commit

Permalink
bigquery: use grpc-grammar.ym
Browse files Browse the repository at this point in the history 
Signed-off-by: Attila Szakacs <[email protected]>
  • Loading branch information
@alltilla
alltilla committed Sep 6, 2024
1 parent 88476b5 commit 7e51356
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 121 deletions.
8 changes: 8 additions & 0 deletions modules/grpc/bigquery/bigquery-dest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@ DestinationDriver::set_protobuf_schema(std::string proto_path, GList *values)
bool
DestinationDriver::init()
{
if (this->batch_bytes > 10 * 1000 * 1000)
{
msg_error("Error initializing BigQuery destination, batch-bytes() cannot be larger than 10 MB. "
"For more info see https://cloud.google.com/bigquery/quotas#write-api-limits",
log_pipe_location_tag(&this->super->super.super.super.super));
return false;
}

GlobalConfig *cfg = log_pipe_get_config(&this->super->super.super.super.super);
log_template_options_init(&this->template_options, cfg);

Expand Down
125 changes: 4 additions & 121 deletions modules/grpc/bigquery/bigquery-grammar.ym
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
*
*/

/* REQUIRE modules/grpc/common/grpc-grammar.ym */

%code top {
#include "bigquery-parser.h"

Expand All @@ -35,12 +37,9 @@
#include "syslog-names.h"

#include "bigquery-dest.h"
#include "grpc-dest.h"

#include <stdlib.h>

GrpcClientCredentialsBuilderW *last_grpc_client_credentials_builder;

}

%define api.prefix {bigquery_}
Expand All @@ -53,32 +52,11 @@ GrpcClientCredentialsBuilderW *last_grpc_client_credentials_builder;
/* INCLUDE_DECLS */

%token KW_BIGQUERY
%token KW_URL
%token KW_PROJECT
%token KW_DATASET
%token KW_TABLE
%token KW_SCHEMA
%token KW_PROTOBUF_SCHEMA
%token KW_BATCH_BYTES
%token KW_COMPRESSION

%token KW_KEEP_ALIVE
%token KW_TIME
%token KW_TIMEOUT
%token KW_MAX_PINGS_WITHOUT_DATA

%token KW_CHANNEL_ARGS
%token KW_HEADERS

%token KW_AUTH
%token KW_INSECURE
%token KW_TLS
%token KW_KEY_FILE
%token KW_CERT_FILE
%token KW_CA_FILE
%token KW_ALTS
%token KW_TARGET_SERVICE_ACCOUNTS
%token KW_ADC

%type <ptr> bigquery_dest

Expand All @@ -102,9 +80,7 @@ bigquery_dest_options
;

bigquery_dest_option
: KW_URL '(' string ')' { grpc_dd_set_url(last_driver, $3); free($3); }
| KW_AUTH { last_grpc_client_credentials_builder = grpc_dd_get_credentials_builder(last_driver); } '(' grpc_client_credentials_option ')'
| KW_PROJECT '(' string ')' { bigquery_dd_set_project(last_driver, $3); free($3); }
: KW_PROJECT '(' string ')' { bigquery_dd_set_project(last_driver, $3); free($3); }
| KW_DATASET '(' string ')' { bigquery_dd_set_dataset(last_driver, $3); free($3); }
| KW_TABLE '(' string ')' { bigquery_dd_set_table(last_driver, $3); free($3); }
| KW_SCHEMA '(' bigquery_schema_fields ')'
Expand All @@ -113,21 +89,7 @@ bigquery_dest_option
bigquery_dd_set_protobuf_schema(last_driver, $3, $5);
free($3);
}
| KW_BATCH_BYTES '(' positive_integer ')'
{
CHECK_ERROR($3 <= (10 * 1000 * 1000), @3, "batch-bytes() cannot be larger than 10 MB. For more info see https://cloud.google.com/bigquery/quotas#write-api-limits");
grpc_dd_set_batch_bytes(last_driver, $3);
}
| KW_COMPRESSION '(' yesno ')'
{
grpc_dd_set_compression(last_driver, $3);
}
| KW_KEEP_ALIVE '(' bigquery_keepalive_options ')'
| KW_CHANNEL_ARGS '(' bigquery_dest_channel_args ')'
| KW_HEADERS '(' bigquery_dest_headers ')'
| threaded_dest_driver_general_option
| threaded_dest_driver_workers_option
| threaded_dest_driver_batch_option
| grpc_dest_option
| { last_template_options = bigquery_dd_get_template_options(last_driver); } template_option
;

Expand All @@ -146,85 +108,6 @@ bigquery_schema_field
}
;

bigquery_keepalive_options
: bigquery_keepalive_option bigquery_keepalive_options
|
;

bigquery_keepalive_option
: KW_TIME '(' nonnegative_integer ')' { grpc_dd_set_keepalive_time(last_driver, $3); }
| KW_TIMEOUT '(' nonnegative_integer ')' { grpc_dd_set_keepalive_timeout(last_driver, $3); }
| KW_MAX_PINGS_WITHOUT_DATA '(' nonnegative_integer ')' { grpc_dd_set_keepalive_max_pings(last_driver, $3); }
;

bigquery_dest_channel_args
: bigquery_dest_channel_arg bigquery_dest_channel_args
|
;

bigquery_dest_channel_arg
: string LL_ARROW LL_NUMBER { grpc_dd_add_int_channel_arg(last_driver, $1, $3); free($1); }
| string LL_ARROW string { grpc_dd_add_string_channel_arg(last_driver, $1, $3); free($1); free($3); }
;

bigquery_dest_headers
: bigquery_dest_header bigquery_dest_headers
|
;

bigquery_dest_header
: string LL_ARROW string { grpc_dd_add_header(last_driver, $1, $3); free($1); free($3); }
;

grpc_client_credentials_option
: KW_INSECURE { grpc_client_credentials_builder_set_mode(last_grpc_client_credentials_builder, GCAM_INSECURE); } '(' ')'
| KW_TLS { grpc_client_credentials_builder_set_mode(last_grpc_client_credentials_builder, GCAM_TLS); } '(' grpc_client_credentials_builder_tls_options ')'
| KW_ALTS { grpc_client_credentials_builder_set_mode(last_grpc_client_credentials_builder, GCAM_ALTS); } '(' grpc_client_credentials_builder_alts_options ')'
| KW_ADC { grpc_client_credentials_builder_set_mode(last_grpc_client_credentials_builder, GCAM_ADC); } '(' ')'
;

grpc_client_credentials_builder_tls_options
: grpc_client_credentials_builder_tls_option grpc_client_credentials_builder_tls_options
|
;

grpc_client_credentials_builder_tls_option
: KW_KEY_FILE '(' string ')'
{
CHECK_ERROR(grpc_client_credentials_builder_set_tls_key_path(last_grpc_client_credentials_builder, $3), @1, "Failed to set key-file()");
free($3);
}
| KW_CERT_FILE '(' string ')'
{
CHECK_ERROR(grpc_client_credentials_builder_set_tls_cert_path(last_grpc_client_credentials_builder, $3), @1, "Failed to set cert-file()");
free($3);
}
| KW_CA_FILE '(' string ')'
{
CHECK_ERROR(grpc_client_credentials_builder_set_tls_ca_path(last_grpc_client_credentials_builder, $3), @1, "Failed to set ca-file()");
free($3);
}
;

grpc_client_credentials_builder_alts_options
: grpc_client_credentials_builder_alts_option grpc_client_credentials_builder_alts_options
|
;

grpc_client_credentials_builder_alts_option
: KW_TARGET_SERVICE_ACCOUNTS '(' grpc_client_credentials_builder_alts_target_service_accounts ')'
;

grpc_client_credentials_builder_alts_target_service_accounts
: string
{
grpc_client_credentials_builder_add_alts_target_service_account(last_grpc_client_credentials_builder, $1);
free($1);
}
grpc_client_credentials_builder_alts_target_service_accounts
|
;

/* INCLUDE_RULES */

%%

0 comments on commit 7e51356

Please sign in to comment.