From 5c1e96f1aa2cbd3df20841a1f41747489b7e392c Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 3 Sep 2024 13:52:35 +0200 Subject: [PATCH 01/25] grpc: move metrics and credentials under common Signed-off-by: Attila Szakacs --- modules/grpc/CMakeLists.txt | 3 +- modules/grpc/Makefile.am | 3 +- modules/grpc/bigquery/CMakeLists.txt | 4 +-- modules/grpc/bigquery/Makefile.am | 5 ++- modules/grpc/common/CMakeLists.txt | 13 ++++++++ modules/grpc/common/Makefile.am | 33 +++++++++++++++++++ .../grpc/common/credentials/CMakeLists.txt | 5 +++ modules/grpc/common/credentials/Makefile.am | 6 ++++ .../credentials/grpc-credentials-builder.cpp | 0 .../credentials/grpc-credentials-builder.h | 0 .../credentials/grpc-credentials-builder.hpp | 0 modules/grpc/common/metrics/CMakeLists.txt | 4 +++ modules/grpc/common/metrics/Makefile.am | 5 +++ .../{ => common}/metrics/grpc-metrics.cpp | 0 .../{ => common}/metrics/grpc-metrics.hpp | 0 modules/grpc/credentials/CMakeLists.txt | 5 --- modules/grpc/credentials/Makefile.am | 6 ---- modules/grpc/loki/CMakeLists.txt | 4 +-- modules/grpc/loki/Makefile.am | 6 ++-- modules/grpc/metrics/CMakeLists.txt | 4 --- modules/grpc/metrics/Makefile.am | 5 --- modules/grpc/otel/CMakeLists.txt | 6 ++-- modules/grpc/otel/Makefile.am | 11 ++++--- tests/copyright/policy | 3 +- 24 files changed, 89 insertions(+), 42 deletions(-) create mode 100644 modules/grpc/common/CMakeLists.txt create mode 100644 modules/grpc/common/Makefile.am create mode 100644 modules/grpc/common/credentials/CMakeLists.txt create mode 100644 modules/grpc/common/credentials/Makefile.am rename modules/grpc/{ => common}/credentials/grpc-credentials-builder.cpp (100%) rename modules/grpc/{ => common}/credentials/grpc-credentials-builder.h (100%) rename modules/grpc/{ => common}/credentials/grpc-credentials-builder.hpp (100%) create mode 100644 modules/grpc/common/metrics/CMakeLists.txt create mode 100644 modules/grpc/common/metrics/Makefile.am rename modules/grpc/{ => common}/metrics/grpc-metrics.cpp (100%) rename modules/grpc/{ => common}/metrics/grpc-metrics.hpp (100%) delete mode 100644 modules/grpc/credentials/CMakeLists.txt delete mode 100644 modules/grpc/credentials/Makefile.am delete mode 100644 modules/grpc/metrics/CMakeLists.txt delete mode 100644 modules/grpc/metrics/Makefile.am diff --git a/modules/grpc/CMakeLists.txt b/modules/grpc/CMakeLists.txt index 13913283ad..6fc3d9174e 100644 --- a/modules/grpc/CMakeLists.txt +++ b/modules/grpc/CMakeLists.txt @@ -70,8 +70,7 @@ if (ENABLE_GRPC) gRPC::grpc++ protobuf::libprotobuf) - add_subdirectory(credentials) - add_subdirectory(metrics) + add_subdirectory(common) add_subdirectory(protos) endif() diff --git a/modules/grpc/Makefile.am b/modules/grpc/Makefile.am index ffab7c0f1a..a46b33f5b1 100644 --- a/modules/grpc/Makefile.am +++ b/modules/grpc/Makefile.am @@ -1,7 +1,6 @@ include modules/grpc/protos/Makefile.am -include modules/grpc/credentials/Makefile.am -include modules/grpc/metrics/Makefile.am +include modules/grpc/common/Makefile.am include modules/grpc/otel/Makefile.am include modules/grpc/loki/Makefile.am diff --git a/modules/grpc/bigquery/CMakeLists.txt b/modules/grpc/bigquery/CMakeLists.txt index e15f79ddce..6e260b7888 100644 --- a/modules/grpc/bigquery/CMakeLists.txt +++ b/modules/grpc/bigquery/CMakeLists.txt @@ -21,7 +21,7 @@ set(BIGQUERY_SOURCES add_module( TARGET bigquery-cpp SOURCES ${BIGQUERY_CPP_SOURCES} - DEPENDS ${MODULE_GRPC_LIBS} grpc-protos + DEPENDS ${MODULE_GRPC_LIBS} grpc-protos grpc-common-cpp INCLUDES ${BIGQUERY_PROTO_BUILDDIR} ${PROJECT_SOURCE_DIR}/modules/grpc LIBRARY_TYPE STATIC ) @@ -29,7 +29,7 @@ add_module( add_module( TARGET bigquery GRAMMAR bigquery-grammar - DEPENDS bigquery-cpp + DEPENDS bigquery-cpp grpc-common-cpp INCLUDES ${PROJECT_SOURCE_DIR}/modules/grpc SOURCES ${BIGQUERY_SOURCES} ) diff --git a/modules/grpc/bigquery/Makefile.am b/modules/grpc/bigquery/Makefile.am index 8eef5f8caf..cc70522ba7 100644 --- a/modules/grpc/bigquery/Makefile.am +++ b/modules/grpc/bigquery/Makefile.am @@ -3,7 +3,6 @@ if ENABLE_GRPC noinst_LTLIBRARIES += modules/grpc/bigquery/libbigquery_cpp.la modules_grpc_bigquery_libbigquery_cpp_la_SOURCES = \ - $(grpc_metrics_sources) \ modules/grpc/bigquery/bigquery-dest.h \ modules/grpc/bigquery/bigquery-dest.hpp \ modules/grpc/bigquery/bigquery-dest.cpp \ @@ -15,6 +14,7 @@ modules_grpc_bigquery_libbigquery_cpp_la_CXXFLAGS = \ $(AM_CXXFLAGS) \ $(PROTOBUF_CFLAGS) \ $(GRPCPP_CFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(GOOGLEAPIS_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/bigquery \ @@ -34,12 +34,14 @@ modules_grpc_bigquery_libbigquery_la_SOURCES = \ modules_grpc_bigquery_libbigquery_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(top_srcdir)/modules/grpc/bigquery \ -I$(top_builddir)/modules/grpc/bigquery \ -I$(top_srcdir)/modules/grpc modules_grpc_bigquery_libbigquery_la_LIBADD = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/bigquery/libbigquery_cpp.la @@ -48,6 +50,7 @@ nodist_EXTRA_modules_grpc_bigquery_libbigquery_la_SOURCES = force-cpp-linker-wit modules_grpc_bigquery_libbigquery_la_LDFLAGS = $(MODULE_LDFLAGS) EXTRA_modules_grpc_bigquery_libbigquery_la_DEPENDENCIES = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/bigquery/libbigquery_cpp.la diff --git a/modules/grpc/common/CMakeLists.txt b/modules/grpc/common/CMakeLists.txt new file mode 100644 index 0000000000..16b57c9b19 --- /dev/null +++ b/modules/grpc/common/CMakeLists.txt @@ -0,0 +1,13 @@ +add_subdirectory(credentials) +add_subdirectory(metrics) + +set(GRPC_COMMON_CPP_SOURCES + ${GRPC_CREDENTIALS_SOURCES} + ${GRPC_METRICS_SOURCES}) + +add_module( + TARGET grpc-common-cpp + SOURCES ${GRPC_COMMON_CPP_SOURCES} + INCLUDES ${PROJECT_SOURCE_DIR}/modules/grpc/common + LIBRARY_TYPE STATIC +) diff --git a/modules/grpc/common/Makefile.am b/modules/grpc/common/Makefile.am new file mode 100644 index 0000000000..565ba778be --- /dev/null +++ b/modules/grpc/common/Makefile.am @@ -0,0 +1,33 @@ +include modules/grpc/common/credentials/Makefile.am +include modules/grpc/common/metrics/Makefile.am + +if ENABLE_GRPC + +noinst_LTLIBRARIES += modules/grpc/common/libgrpc-common.la + +GRPC_COMMON_CFLAGS = \ + -I$(top_srcdir)/modules/grpc/common \ + -I$(top_builddir)/modules/grpc/common + +GRPC_COMMON_LIBS = \ + $(top_builddir)/modules/grpc/common/libgrpc-common.la + + +modules_grpc_common_libgrpc_common_la_SOURCES = \ + $(grpc_credentials_sources) \ + $(grpc_metrics_sources) + +modules_grpc_common_libgrpc_common_la_CXXFLAGS = \ + $(AM_CXXFLAGS) \ + $(PROTOBUF_CFLAGS) \ + $(GRPCPP_CFLAGS) \ + $(GRPC_COMMON_CFLAGS) + +modules_grpc_common_libgrpc_common_la_LIBADD = $(MODULE_DEPS_LIBS) $(PROTOBUF_LIBS) $(GRPCPP_LIBS) +modules_grpc_common_libgrpc_common_la_LDFLAGS = $(MODULE_LDFLAGS) +EXTRA_modules_grpc_common_libgrpc_common_la_DEPENDENCIES = $(MODULE_DEPS_LIBS) + +endif + +EXTRA_DIST += \ + modules/grpc/common/CMakeLists.txt diff --git a/modules/grpc/common/credentials/CMakeLists.txt b/modules/grpc/common/credentials/CMakeLists.txt new file mode 100644 index 0000000000..23da5f42fa --- /dev/null +++ b/modules/grpc/common/credentials/CMakeLists.txt @@ -0,0 +1,5 @@ +set(GRPC_CREDENTIALS_SOURCES + ${PROJECT_SOURCE_DIR}/modules/grpc/common/credentials/grpc-credentials-builder.h + ${PROJECT_SOURCE_DIR}/modules/grpc/common/credentials/grpc-credentials-builder.hpp + ${PROJECT_SOURCE_DIR}/modules/grpc/common/credentials/grpc-credentials-builder.cpp + PARENT_SCOPE) diff --git a/modules/grpc/common/credentials/Makefile.am b/modules/grpc/common/credentials/Makefile.am new file mode 100644 index 0000000000..67f1867986 --- /dev/null +++ b/modules/grpc/common/credentials/Makefile.am @@ -0,0 +1,6 @@ +grpc_credentials_sources = \ + modules/grpc/common/credentials/grpc-credentials-builder.h \ + modules/grpc/common/credentials/grpc-credentials-builder.hpp \ + modules/grpc/common/credentials/grpc-credentials-builder.cpp + +EXTRA_DIST += modules/grpc/common/credentials/CMakeLists.txt diff --git a/modules/grpc/credentials/grpc-credentials-builder.cpp b/modules/grpc/common/credentials/grpc-credentials-builder.cpp similarity index 100% rename from modules/grpc/credentials/grpc-credentials-builder.cpp rename to modules/grpc/common/credentials/grpc-credentials-builder.cpp diff --git a/modules/grpc/credentials/grpc-credentials-builder.h b/modules/grpc/common/credentials/grpc-credentials-builder.h similarity index 100% rename from modules/grpc/credentials/grpc-credentials-builder.h rename to modules/grpc/common/credentials/grpc-credentials-builder.h diff --git a/modules/grpc/credentials/grpc-credentials-builder.hpp b/modules/grpc/common/credentials/grpc-credentials-builder.hpp similarity index 100% rename from modules/grpc/credentials/grpc-credentials-builder.hpp rename to modules/grpc/common/credentials/grpc-credentials-builder.hpp diff --git a/modules/grpc/common/metrics/CMakeLists.txt b/modules/grpc/common/metrics/CMakeLists.txt new file mode 100644 index 0000000000..17359bad24 --- /dev/null +++ b/modules/grpc/common/metrics/CMakeLists.txt @@ -0,0 +1,4 @@ +set(GRPC_METRICS_SOURCES + ${PROJECT_SOURCE_DIR}/modules/grpc/common/metrics/grpc-metrics.hpp + ${PROJECT_SOURCE_DIR}/modules/grpc/common/metrics/grpc-metrics.cpp + PARENT_SCOPE) diff --git a/modules/grpc/common/metrics/Makefile.am b/modules/grpc/common/metrics/Makefile.am new file mode 100644 index 0000000000..2832b5c0d9 --- /dev/null +++ b/modules/grpc/common/metrics/Makefile.am @@ -0,0 +1,5 @@ +grpc_metrics_sources = \ + modules/grpc/common/metrics/grpc-metrics.hpp \ + modules/grpc/common/metrics/grpc-metrics.cpp + +EXTRA_DIST += modules/grpc/common/metrics/CMakeLists.txt diff --git a/modules/grpc/metrics/grpc-metrics.cpp b/modules/grpc/common/metrics/grpc-metrics.cpp similarity index 100% rename from modules/grpc/metrics/grpc-metrics.cpp rename to modules/grpc/common/metrics/grpc-metrics.cpp diff --git a/modules/grpc/metrics/grpc-metrics.hpp b/modules/grpc/common/metrics/grpc-metrics.hpp similarity index 100% rename from modules/grpc/metrics/grpc-metrics.hpp rename to modules/grpc/common/metrics/grpc-metrics.hpp diff --git a/modules/grpc/credentials/CMakeLists.txt b/modules/grpc/credentials/CMakeLists.txt deleted file mode 100644 index df5ac44590..0000000000 --- a/modules/grpc/credentials/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(GRPC_CREDENTIALS_SOURCES - ${PROJECT_SOURCE_DIR}/modules/grpc/credentials/grpc-credentials-builder.h - ${PROJECT_SOURCE_DIR}/modules/grpc/credentials/grpc-credentials-builder.hpp - ${PROJECT_SOURCE_DIR}/modules/grpc/credentials/grpc-credentials-builder.cpp - PARENT_SCOPE) diff --git a/modules/grpc/credentials/Makefile.am b/modules/grpc/credentials/Makefile.am deleted file mode 100644 index 31c5e07710..0000000000 --- a/modules/grpc/credentials/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -grpc_credentials_sources = \ - modules/grpc/credentials/grpc-credentials-builder.h \ - modules/grpc/credentials/grpc-credentials-builder.hpp \ - modules/grpc/credentials/grpc-credentials-builder.cpp - -EXTRA_DIST += modules/grpc/credentials/CMakeLists.txt diff --git a/modules/grpc/loki/CMakeLists.txt b/modules/grpc/loki/CMakeLists.txt index 561709c7f1..7136edf142 100644 --- a/modules/grpc/loki/CMakeLists.txt +++ b/modules/grpc/loki/CMakeLists.txt @@ -22,7 +22,7 @@ set(LOKI_SOURCES add_module( TARGET loki-cpp SOURCES ${LOKI_CPP_SOURCES} - DEPENDS ${MODULE_GRPC_LIBS} grpc-protos + DEPENDS ${MODULE_GRPC_LIBS} grpc-protos grpc-common-cpp INCLUDES ${LOKI_PROTO_BUILDDIR} ${PROJECT_SOURCE_DIR}/modules/grpc LIBRARY_TYPE STATIC ) @@ -30,7 +30,7 @@ add_module( add_module( TARGET loki GRAMMAR loki-grammar - DEPENDS loki-cpp + DEPENDS loki-cpp grpc-common-cpp INCLUDES ${PROJECT_SOURCE_DIR}/modules/grpc SOURCES ${LOKI_SOURCES} ) diff --git a/modules/grpc/loki/Makefile.am b/modules/grpc/loki/Makefile.am index 178a6f7d87..37aefd703c 100644 --- a/modules/grpc/loki/Makefile.am +++ b/modules/grpc/loki/Makefile.am @@ -3,8 +3,6 @@ if ENABLE_GRPC noinst_LTLIBRARIES += modules/grpc/loki/libloki_cpp.la modules_grpc_loki_libloki_cpp_la_SOURCES = \ - $(grpc_credentials_sources) \ - $(grpc_metrics_sources) \ modules/grpc/loki/loki-dest.h \ modules/grpc/loki/loki-dest.hpp \ modules/grpc/loki/loki-dest.cpp \ @@ -16,6 +14,7 @@ modules_grpc_loki_libloki_cpp_la_CXXFLAGS = \ $(AM_CXXFLAGS) \ $(PROTOBUF_CFLAGS) \ $(GRPCPP_CFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(LOKI_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/loki \ @@ -35,6 +34,7 @@ modules_grpc_loki_libloki_la_SOURCES = \ modules_grpc_loki_libloki_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/loki \ -I$(top_builddir)/modules/grpc/loki \ @@ -42,6 +42,7 @@ modules_grpc_loki_libloki_la_CPPFLAGS = \ modules_grpc_loki_libloki_la_LIBADD = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/loki/libloki_cpp.la @@ -50,6 +51,7 @@ nodist_EXTRA_modules_grpc_loki_libloki_la_SOURCES = force-cpp-linker-with-defaul modules_grpc_loki_libloki_la_LDFLAGS = $(MODULE_LDFLAGS) EXTRA_modules_grpc_loki_libloki_la_DEPENDENCIES = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/loki/libloki_cpp.la diff --git a/modules/grpc/metrics/CMakeLists.txt b/modules/grpc/metrics/CMakeLists.txt deleted file mode 100644 index 54eb2e6ae5..0000000000 --- a/modules/grpc/metrics/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(GRPC_METRICS_SOURCES - ${PROJECT_SOURCE_DIR}/modules/grpc/metrics/grpc-metrics.hpp - ${PROJECT_SOURCE_DIR}/modules/grpc/metrics/grpc-metrics.cpp - PARENT_SCOPE) diff --git a/modules/grpc/metrics/Makefile.am b/modules/grpc/metrics/Makefile.am deleted file mode 100644 index 0d6d9d9eb3..0000000000 --- a/modules/grpc/metrics/Makefile.am +++ /dev/null @@ -1,5 +0,0 @@ -grpc_metrics_sources = \ - modules/grpc/metrics/grpc-metrics.hpp \ - modules/grpc/metrics/grpc-metrics.cpp - -EXTRA_DIST += modules/grpc/metrics/CMakeLists.txt diff --git a/modules/grpc/otel/CMakeLists.txt b/modules/grpc/otel/CMakeLists.txt index 9fcfdda6b7..d6d49a5e1f 100644 --- a/modules/grpc/otel/CMakeLists.txt +++ b/modules/grpc/otel/CMakeLists.txt @@ -3,8 +3,6 @@ if(NOT ENABLE_GRPC) endif() set(OTEL_CPP_SOURCES - ${GRPC_CREDENTIALS_SOURCES} - ${GRPC_METRICS_SOURCES} otel-source.cpp otel-source.hpp otel-source.h @@ -38,7 +36,7 @@ set(OTEL_SOURCES add_module( TARGET otel-cpp SOURCES ${OTEL_CPP_SOURCES} - DEPENDS ${MODULE_GRPC_LIBS} grpc-protos + DEPENDS ${MODULE_GRPC_LIBS} grpc-protos grpc-common-cpp INCLUDES ${OTEL_PROTO_BUILDDIR} ${PROJECT_SOURCE_DIR}/modules/grpc LIBRARY_TYPE STATIC ) @@ -46,7 +44,7 @@ add_module( add_module( TARGET otel GRAMMAR otel-grammar - DEPENDS otel-cpp + DEPENDS otel-cpp grpc-common-cpp INCLUDES ${PROJECT_SOURCE_DIR}/modules/grpc SOURCES ${OTEL_SOURCES} ) diff --git a/modules/grpc/otel/Makefile.am b/modules/grpc/otel/Makefile.am index 5ece1b33f9..acc1298ca4 100644 --- a/modules/grpc/otel/Makefile.am +++ b/modules/grpc/otel/Makefile.am @@ -3,8 +3,6 @@ if ENABLE_GRPC noinst_LTLIBRARIES += modules/grpc/otel/libotel_cpp.la modules_grpc_otel_libotel_cpp_la_SOURCES = \ - $(grpc_credentials_sources) \ - $(grpc_metrics_sources) \ modules/grpc/otel/otel-source.h \ modules/grpc/otel/otel-source.hpp \ modules/grpc/otel/otel-source.cpp \ @@ -33,13 +31,14 @@ modules_grpc_otel_libotel_cpp_la_CXXFLAGS = \ $(AM_CXXFLAGS) \ $(PROTOBUF_CFLAGS) \ $(GRPCPP_CFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(OPENTELEMETRY_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/otel \ -I$(top_builddir)/modules/grpc/otel modules_grpc_otel_libotel_cpp_la_LIBADD = $(MODULE_DEPS_LIBS) $(PROTOBUF_LIBS) $(GRPCPP_LIBS) -modules_grpc_otel_libotel_cpp_la_LDFLAGS = $(MODULE_LDFLAGS) +modules_grpc_otel_libotel_cpp_la_LDFLAGS = $(MODULE_LDFLAGS) EXTRA_modules_grpc_otel_libotel_cpp_la_DEPENDENCIES = $(MODULE_DEPS_LIBS) module_LTLIBRARIES += modules/grpc/otel/libotel.la @@ -52,13 +51,14 @@ modules_grpc_otel_libotel_la_SOURCES = \ modules_grpc_otel_libotel_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ + $(GRPC_COMMON_CFLAGS) \ -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/otel \ - -I$(top_builddir)/modules/grpc/otel \ - -I$(top_srcdir)/modules/grpc + -I$(top_builddir)/modules/grpc/otel modules_grpc_otel_libotel_la_LIBADD = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ $(top_builddir)/modules/grpc/otel/filterx/libfilterx.la @@ -69,6 +69,7 @@ modules_grpc_otel_libotel_la_LDFLAGS = $(MODULE_LDFLAGS) EXTRA_modules_grpc_otel_libotel_la_DEPENDENCIES = \ $(MODULE_DEPS_LIBS) \ + $(GRPC_COMMON_LIBS) \ $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ $(top_builddir)/modules/grpc/otel/filterx/libfilterx.la diff --git a/tests/copyright/policy b/tests/copyright/policy index 0b3a44acd5..c40c166e24 100644 --- a/tests/copyright/policy +++ b/tests/copyright/policy @@ -209,8 +209,7 @@ modules/grpc/otel/tests/test-syslog-ng-otlp\.cpp$ modules/grpc/otel/tests/test-otel-filterx\.cpp$ modules/grpc/loki modules/grpc/bigquery -modules/grpc/credentials -modules/grpc/metrics +modules/grpc/common modules/grpc/protos/apphook\.(cpp|h)$ modules/cloud-auth/cloud-auth(|-grammar|-parser|-plugin)\.(c|h|cpp|hpp|ym)$ modules/cloud-auth/google-auth\.(h|cpp|hpp)$ From f23a392cbddf8c45368eed2ce6e4a86fbd495153 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 3 Sep 2024 15:41:55 +0200 Subject: [PATCH 02/25] grpc: create DestDriver and DestWorker base classes Signed-off-by: Attila Szakacs --- modules/grpc/common/CMakeLists.txt | 7 +- modules/grpc/common/Makefile.am | 7 +- modules/grpc/common/grpc-dest-worker.cpp | 168 +++++++++++++++++++ modules/grpc/common/grpc-dest-worker.hpp | 73 ++++++++ modules/grpc/common/grpc-dest.cpp | 205 +++++++++++++++++++++++ modules/grpc/common/grpc-dest.h | 50 ++++++ modules/grpc/common/grpc-dest.hpp | 159 ++++++++++++++++++ 7 files changed, 667 insertions(+), 2 deletions(-) create mode 100644 modules/grpc/common/grpc-dest-worker.cpp create mode 100644 modules/grpc/common/grpc-dest-worker.hpp create mode 100644 modules/grpc/common/grpc-dest.cpp create mode 100644 modules/grpc/common/grpc-dest.h create mode 100644 modules/grpc/common/grpc-dest.hpp diff --git a/modules/grpc/common/CMakeLists.txt b/modules/grpc/common/CMakeLists.txt index 16b57c9b19..f77c67ced8 100644 --- a/modules/grpc/common/CMakeLists.txt +++ b/modules/grpc/common/CMakeLists.txt @@ -3,7 +3,12 @@ add_subdirectory(metrics) set(GRPC_COMMON_CPP_SOURCES ${GRPC_CREDENTIALS_SOURCES} - ${GRPC_METRICS_SOURCES}) + ${GRPC_METRICS_SOURCES} + grpc-dest.hpp + grpc-dest.cpp + grpc-dest.h + grpc-dest-worker.hpp + grpc-dest-worker.cpp) add_module( TARGET grpc-common-cpp diff --git a/modules/grpc/common/Makefile.am b/modules/grpc/common/Makefile.am index 565ba778be..eb6dadf21e 100644 --- a/modules/grpc/common/Makefile.am +++ b/modules/grpc/common/Makefile.am @@ -15,7 +15,12 @@ GRPC_COMMON_LIBS = \ modules_grpc_common_libgrpc_common_la_SOURCES = \ $(grpc_credentials_sources) \ - $(grpc_metrics_sources) + $(grpc_metrics_sources) \ + modules/grpc/common/grpc-dest.h \ + modules/grpc/common/grpc-dest.hpp \ + modules/grpc/common/grpc-dest.cpp \ + modules/grpc/common/grpc-dest-worker.hpp \ + modules/grpc/common/grpc-dest-worker.cpp modules_grpc_common_libgrpc_common_la_CXXFLAGS = \ $(AM_CXXFLAGS) \ diff --git a/modules/grpc/common/grpc-dest-worker.cpp b/modules/grpc/common/grpc-dest-worker.cpp new file mode 100644 index 0000000000..d11652b971 --- /dev/null +++ b/modules/grpc/common/grpc-dest-worker.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023-2024 Attila Szakacs + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include +#include +#include +#include + +#include "grpc-dest-worker.hpp" + +using namespace syslogng::grpc; + +/* C++ Implementations */ + +DestWorker::DestWorker(GrpcDestWorker *s) + : super(s), + owner(*(reinterpret_cast(s->super.owner))->cpp) +{ +} + +std::shared_ptr<::grpc::ChannelCredentials> +DestWorker::create_credentials() +{ + return this->owner.credentials_builder.build(); +} + +::grpc::ChannelArguments +DestWorker::create_channel_args() +{ + ::grpc::ChannelArguments args; + + if (this->owner.keepalive_time != -1) + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, this->owner.keepalive_time); + if (this->owner.keepalive_timeout != -1) + args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, this->owner.keepalive_timeout); + if (this->owner.keepalive_max_pings_without_data != -1) + args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, this->owner.keepalive_max_pings_without_data); + + if (this->owner.compression) + args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); + + args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + + for (auto nv : this->owner.int_extra_channel_args) + args.SetInt(nv.first, nv.second); + for (auto nv : this->owner.string_extra_channel_args) + args.SetString(nv.first, nv.second); + + return args; +} + +bool +DestWorker::init() +{ + return log_threaded_dest_worker_init_method(&super->super); +} + +void +DestWorker::deinit() +{ +} + +bool +DestWorker::connect() +{ + return true; +} + +void +DestWorker::disconnect() +{ +} + +void +DestWorker::prepare_context(::grpc::ClientContext &context) +{ + for (auto nv : owner.headers) + context.AddMetadata(nv.first, nv.second); +} + +/* C Wrappers */ + +static gboolean +_init(LogThreadedDestWorker *s) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + return self->cpp->init(); +} + +static void +_deinit(LogThreadedDestWorker *s) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + self->cpp->deinit(); +} + +static gboolean +_connect(LogThreadedDestWorker *s) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + return self->cpp->connect(); +} + +static void +_disconnect(LogThreadedDestWorker *s) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + self->cpp->disconnect(); +} + +LogThreadedResult +_insert(LogThreadedDestWorker *s, LogMessage *msg) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + return self->cpp->insert(msg); +} + +LogThreadedResult +_flush(LogThreadedDestWorker *s, LogThreadedFlushMode mode) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + return self->cpp->flush(mode); +} + +static void +_free(LogThreadedDestWorker *s) +{ + GrpcDestWorker *self = (GrpcDestWorker *) s; + delete self->cpp; + log_threaded_dest_worker_free_method(s); +} + +GrpcDestWorker * +grpc_dw_new(GrpcDestDriver *o, gint worker_index) +{ + GrpcDestWorker *self = g_new0(GrpcDestWorker, 1); + + log_threaded_dest_worker_init_instance(&self->super, &o->super, worker_index); + self->super.init = _init; + self->super.deinit = _deinit; + self->super.connect = _connect; + self->super.disconnect = _disconnect; + self->super.insert = _insert; + self->super.flush = _flush; + self->super.free_fn = _free; + + return self; +} diff --git a/modules/grpc/common/grpc-dest-worker.hpp b/modules/grpc/common/grpc-dest-worker.hpp new file mode 100644 index 0000000000..ea4ee43e61 --- /dev/null +++ b/modules/grpc/common/grpc-dest-worker.hpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023-2024 Attila Szakacs + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef GRPC_DEST_WORKER_HPP +#define GRPC_DEST_WORKER_HPP + +#include +#include +#include +#include + +#include "grpc-dest.hpp" + +typedef struct GrpcDestWorker_ GrpcDestWorker; + +namespace syslogng { +namespace grpc { + +class DestWorker +{ +public: + DestWorker(GrpcDestWorker *s); + virtual ~DestWorker() {}; + + virtual bool init(); + virtual void deinit(); + virtual bool connect(); + virtual void disconnect(); + virtual LogThreadedResult insert(LogMessage *msg) = 0; + virtual LogThreadedResult flush(LogThreadedFlushMode mode) = 0; + +protected: + void prepare_context(::grpc::ClientContext &context); + std::shared_ptr<::grpc::ChannelCredentials> create_credentials(); + ::grpc::ChannelArguments create_channel_args(); + +protected: + GrpcDestWorker *super; + DestDriver &owner; +}; + +} +} + +GrpcDestWorker *grpc_dw_new(GrpcDestDriver *o, gint worker_index); + +struct GrpcDestWorker_ +{ + LogThreadedDestWorker super; + syslogng::grpc::DestWorker *cpp; +}; + +#endif diff --git a/modules/grpc/common/grpc-dest.cpp b/modules/grpc/common/grpc-dest.cpp new file mode 100644 index 0000000000..2d2baba3de --- /dev/null +++ b/modules/grpc/common/grpc-dest.cpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023-2024 Attila Szakacs + * Copyright (c) 2023 László Várady + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "grpc-dest.hpp" +#include "grpc-dest-worker.hpp" + +using namespace syslogng::grpc; + +/* C++ Implementations */ + +DestDriver::DestDriver(GrpcDestDriver *s) + : super(s), compression(false), batch_bytes(4 * 1000 * 1000) +{ + credentials_builder_wrapper.self = &credentials_builder; +} + +bool +DestDriver::init() +{ + if (url.length() == 0) + { + msg_error("url() option is mandatory", + log_pipe_location_tag(&super->super.super.super.super)); + return false; + } + + if (!credentials_builder.validate()) + { + return false; + } + + if (!log_threaded_dest_driver_init_method(&this->super->super.super.super.super)) + return false; + + log_threaded_dest_driver_register_aggregated_stats(&this->super->super); + + StatsClusterKeyBuilder *kb = stats_cluster_key_builder_new(); + format_stats_key(kb); + metrics.init(kb, log_pipe_is_internal(&super->super.super.super.super) ? STATS_LEVEL3 : STATS_LEVEL1); + + return true; +} + +bool +DestDriver::deinit() +{ + metrics.deinit(); + return log_threaded_dest_driver_deinit_method(&super->super.super.super.super); +} + +/* C Wrappers */ + +void +grpc_dd_set_url(LogDriver *s, const gchar *url) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_url(url); +} + +void +grpc_dd_set_compression(LogDriver *s, gboolean enable) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_compression(enable); +} + +void +grpc_dd_set_batch_bytes(LogDriver *s, glong b) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_batch_bytes((size_t) b); +} + +void +grpc_dd_set_keepalive_time(LogDriver *s, gint t) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_keepalive_time(t); +} + +void +grpc_dd_set_keepalive_timeout(LogDriver *s, gint t) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_keepalive_timeout(t); +} + +void +grpc_dd_set_keepalive_max_pings(LogDriver *s, gint p) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->set_keepalive_max_pings(p); +} + +void +grpc_dd_add_int_channel_arg(LogDriver *s, const gchar *name, glong value) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->add_extra_channel_arg(name, value); +} + +void +grpc_dd_add_string_channel_arg(LogDriver *s, const gchar *name, const gchar *value) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->add_extra_channel_arg(name, value); +} + +void +grpc_dd_add_header(LogDriver *s, const gchar *name, const gchar *value) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + self->cpp->add_header(name, value); +} + +GrpcClientCredentialsBuilderW * +grpc_dd_get_credentials_builder(LogDriver *s) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->get_credentials_builder_wrapper(); +} + +static const gchar * +_format_stats_key(LogThreadedDestDriver *s, StatsClusterKeyBuilder *kb) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->format_stats_key(kb); +} + +static const gchar * +_generate_persist_name(const LogPipe *s) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->generate_persist_name(); +} + +static gboolean +_init(LogPipe *s) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->init(); +} + +static gboolean +_deinit(LogPipe *s) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->deinit(); +} + +static LogThreadedDestWorker * +_construct_worker(LogThreadedDestDriver *s, gint worker_index) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + return self->cpp->construct_worker(worker_index); +} + +static void +_free(LogPipe *s) +{ + GrpcDestDriver *self = (GrpcDestDriver *) s; + delete self->cpp; + log_threaded_dest_driver_free(s); +} + +GrpcDestDriver * +grpc_dd_new(GlobalConfig *cfg, const gchar *stats_name) +{ + GrpcDestDriver *self = g_new0(GrpcDestDriver, 1); + + log_threaded_dest_driver_init_instance(&self->super, cfg); + + self->super.super.super.super.init = _init; + self->super.super.super.super.deinit = _deinit; + self->super.super.super.super.free_fn = _free; + self->super.super.super.super.generate_persist_name = _generate_persist_name; + + self->super.worker.construct = _construct_worker; + self->super.stats_source = stats_register_type(stats_name); + self->super.format_stats_key = _format_stats_key; + self->super.metrics.raw_bytes_enabled = TRUE; + + return self; +} diff --git a/modules/grpc/common/grpc-dest.h b/modules/grpc/common/grpc-dest.h new file mode 100644 index 0000000000..9c051cb667 --- /dev/null +++ b/modules/grpc/common/grpc-dest.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023-2024 Attila Szakacs + * Copyright (c) 2023 László Várady + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef GRPC_DEST_H +#define GRPC_DEST_H + +#include "syslog-ng.h" + +#include "compat/cpp-start.h" + +#include "driver.h" +#include "credentials/grpc-credentials-builder.h" + +typedef struct GrpcDestDriver_ GrpcDestDriver; + +void grpc_dd_set_url(LogDriver *s, const gchar *url); +void grpc_dd_set_compression(LogDriver *s, gboolean enable); +void grpc_dd_set_batch_bytes(LogDriver *s, glong b); +void grpc_dd_set_keepalive_time(LogDriver *s, gint t); +void grpc_dd_set_keepalive_timeout(LogDriver *s, gint t); +void grpc_dd_set_keepalive_max_pings(LogDriver *s, gint p); +void grpc_dd_add_int_channel_arg(LogDriver *s, const gchar *name, glong value); +void grpc_dd_add_string_channel_arg(LogDriver *s, const gchar *name, const gchar *value); +void grpc_dd_add_header(LogDriver *s, const gchar *name, const gchar *value); +GrpcClientCredentialsBuilderW *grpc_dd_get_credentials_builder(LogDriver *s); + +#include "compat/cpp-end.h" + +#endif diff --git a/modules/grpc/common/grpc-dest.hpp b/modules/grpc/common/grpc-dest.hpp new file mode 100644 index 0000000000..a1df173502 --- /dev/null +++ b/modules/grpc/common/grpc-dest.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2023-2024 Attila Szakacs + * Copyright (c) 2023 László Várady + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef GRPC_DEST_HPP +#define GRPC_DEST_HPP + +#include "grpc-dest.h" + +#include "compat/cpp-start.h" +#include "logthrdest/logthrdestdrv.h" +#include "compat/cpp-end.h" + +#include "credentials/grpc-credentials-builder.hpp" +#include "metrics/grpc-metrics.hpp" + +#include + +#include + +namespace syslogng { +namespace grpc { + +class DestDriver +{ +public: + DestDriver(GrpcDestDriver *s); + virtual ~DestDriver() {}; + + virtual bool init(); + virtual bool deinit(); + virtual const char *format_stats_key(StatsClusterKeyBuilder *kb) = 0; + virtual const char *generate_persist_name() = 0; + virtual LogThreadedDestWorker *construct_worker(int worker_index) = 0; + + void set_url(const char *u) + { + this->url.assign(u); + } + + const std::string &get_url() const + { + return this->url; + } + + void set_compression(bool c) + { + this->compression = c; + } + + bool get_compression() const + { + return this->compression; + } + + void set_batch_bytes(size_t b) + { + this->batch_bytes = b; + } + + size_t get_batch_bytes() const + { + return this->batch_bytes; + } + + void set_keepalive_time(int t) + { + this->keepalive_time = t; + } + + void set_keepalive_timeout(int t) + { + this->keepalive_timeout = t; + } + + void set_keepalive_max_pings(int p) + { + this->keepalive_max_pings_without_data = p; + } + + void add_extra_channel_arg(std::string name, long value) + { + this->int_extra_channel_args.push_back(std::pair {name, value}); + } + + void add_extra_channel_arg(std::string name, std::string value) + { + this->string_extra_channel_args.push_back(std::pair {name, value}); + } + + void add_header(std::string name, std::string value) + { + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + this->headers.push_back(std::pair {name, value}); + } + + GrpcClientCredentialsBuilderW *get_credentials_builder_wrapper() + { + return &this->credentials_builder_wrapper; + } + +public: + GrpcDestDriver *super; + DestDriverMetrics metrics; + syslogng::grpc::ClientCredentialsBuilder credentials_builder; + +protected: + friend class DestWorker; + std::string url; + + bool compression; + size_t batch_bytes; + + int keepalive_time; + int keepalive_timeout; + int keepalive_max_pings_without_data; + + std::list> int_extra_channel_args; + std::list> string_extra_channel_args; + + std::list> headers; + + GrpcClientCredentialsBuilderW credentials_builder_wrapper; +}; + +} +} + +struct GrpcDestDriver_ +{ + LogThreadedDestDriver super; + syslogng::grpc::DestDriver *cpp; +}; + +GrpcDestDriver *grpc_dd_new(GlobalConfig *cfg, const gchar *stats_name); + +#endif + + From 0ca92899b49dc235d4a21df9ad7bf268019a850d Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 3 Sep 2024 15:43:59 +0200 Subject: [PATCH 03/25] otel: derive DestDriver and DestWorker from grpc base classes Signed-off-by: Attila Szakacs --- modules/grpc/otel/otel-dest-worker.cpp | 122 +--------- modules/grpc/otel/otel-dest-worker.hpp | 33 +-- modules/grpc/otel/otel-dest.cpp | 209 +----------------- modules/grpc/otel/otel-dest.h | 7 - modules/grpc/otel/otel-dest.hpp | 52 +---- modules/grpc/otel/otel-grammar.ym | 15 +- .../grpc/otel/syslog-ng-otlp-dest-worker.cpp | 11 - .../grpc/otel/syslog-ng-otlp-dest-worker.hpp | 3 - modules/grpc/otel/syslog-ng-otlp-dest.cpp | 10 +- modules/grpc/otel/syslog-ng-otlp-dest.h | 2 - 10 files changed, 32 insertions(+), 432 deletions(-) diff --git a/modules/grpc/otel/otel-dest-worker.cpp b/modules/grpc/otel/otel-dest-worker.cpp index ac01bfacb9..0318c37f6f 100644 --- a/modules/grpc/otel/otel-dest-worker.cpp +++ b/modules/grpc/otel/otel-dest-worker.cpp @@ -31,8 +31,6 @@ #include "otel-dest-worker.hpp" -#define get_DestWorker(s) (((OtelDestWorker *) s)->cpp) - using namespace syslogng::grpc::otel; using namespace google::protobuf::util; using namespace opentelemetry::proto::logs::v1; @@ -41,55 +39,22 @@ using namespace opentelemetry::proto::trace::v1; /* C++ Implementations */ -DestWorker::DestWorker(OtelDestWorker *s) - : super(s), - owner(*((OtelDestDriver *) s->super.owner)->cpp), +DestWorker::DestWorker(GrpcDestWorker *s) + : syslogng::grpc::DestWorker(s), logs_current_batch_bytes(0), metrics_current_batch_bytes(0), spans_current_batch_bytes(0), formatter(s->super.owner->super.super.super.cfg) { - ::grpc::ChannelArguments args; - - if (owner.get_compression()) - { - args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); - } + std::shared_ptr<::grpc::ChannelCredentials> credentials = DestWorker::create_credentials(); + ::grpc::ChannelArguments args = this->create_channel_args(); - for (auto nv : owner.int_extra_channel_args) - args.SetInt(nv.first, nv.second); - for (auto nv : owner.string_extra_channel_args) - args.SetString(nv.first, nv.second); - - channel = ::grpc::CreateCustomChannel(owner.get_url(), owner.credentials_builder.build(), args); + channel = ::grpc::CreateCustomChannel(owner.get_url(), credentials, args); logs_service_stub = LogsService::NewStub(channel); metrics_service_stub = MetricsService::NewStub(channel); trace_service_stub = TraceService::NewStub(channel); } -bool -DestWorker::init() -{ - return log_threaded_dest_worker_init_method(&super->super); -} - -void -DestWorker::deinit() -{ - log_threaded_dest_worker_deinit_method(&super->super); -} - -bool -DestWorker::connect() -{ - return true; -} - -void -DestWorker::disconnect() -{ -} - void DestWorker::clear_current_msg_metadata() { @@ -432,13 +397,6 @@ _map_grpc_status_to_log_threaded_result(const ::grpc::Status &status) return LTR_DROP; } -void -DestWorker::prepare_context(::grpc::ClientContext &context) -{ - for (auto nv : owner.headers) - context.AddMetadata(nv.first, nv.second); -} - LogThreadedResult DestWorker::flush_log_records() { @@ -541,73 +499,3 @@ DestWorker::flush(LogThreadedFlushMode mode) return result; } - -/* C Wrappers */ - -static gboolean -_init(LogThreadedDestWorker *s) -{ - return get_DestWorker(s)->init(); -} - -static void -_deinit(LogThreadedDestWorker *s) -{ - get_DestWorker(s)->deinit(); -} - -static gboolean -_connect(LogThreadedDestWorker *s) -{ - return get_DestWorker(s)->connect(); -} - -static void -_disconnect(LogThreadedDestWorker *s) -{ - get_DestWorker(s)->disconnect(); -} - -LogThreadedResult -_insert(LogThreadedDestWorker *s, LogMessage *msg) -{ - return get_DestWorker(s)->insert(msg); -} - -LogThreadedResult -_flush(LogThreadedDestWorker *s, LogThreadedFlushMode mode) -{ - return get_DestWorker(s)->flush(mode); -} - -static void -_free(LogThreadedDestWorker *s) -{ - delete get_DestWorker(s); - log_threaded_dest_worker_free_method(s); -} - -void -otel_dw_init_super(LogThreadedDestWorker *s, LogThreadedDestDriver *o, gint worker_index) -{ - log_threaded_dest_worker_init_instance(s, o, worker_index); - - s->init = _init; - s->deinit = _deinit; - s->connect = _connect; - s->disconnect = _disconnect; - s->insert = _insert; - s->flush = _flush; - s->free_fn = _free; -} - -LogThreadedDestWorker * -DestWorker::construct(LogThreadedDestDriver *o, gint worker_index) -{ - OtelDestWorker *self = g_new0(OtelDestWorker, 1); - - otel_dw_init_super(&self->super, o, worker_index); - self->cpp = new DestWorker(self); - - return &self->super; -} diff --git a/modules/grpc/otel/otel-dest-worker.hpp b/modules/grpc/otel/otel-dest-worker.hpp index 5f32957fca..62f171b068 100644 --- a/modules/grpc/otel/otel-dest-worker.hpp +++ b/modules/grpc/otel/otel-dest-worker.hpp @@ -24,9 +24,6 @@ #ifndef OTEL_DEST_WORKER_HPP #define OTEL_DEST_WORKER_HPP -#include -#include - #include "opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.h" #include "opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h" #include "opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.h" @@ -34,11 +31,10 @@ #include "opentelemetry/proto/metrics/v1/metrics.pb.h" #include "opentelemetry/proto/trace/v1/trace.pb.h" +#include "grpc-dest-worker.hpp" #include "otel-dest.hpp" #include "otel-protobuf-formatter.hpp" -typedef struct OtelDestWorker_ OtelDestWorker; - namespace syslogng { namespace grpc { namespace otel { @@ -56,23 +52,15 @@ using opentelemetry::proto::logs::v1::ScopeLogs; using opentelemetry::proto::metrics::v1::ScopeMetrics; using opentelemetry::proto::trace::v1::ScopeSpans; -class DestWorker +class DestWorker : public syslogng::grpc::DestWorker { public: - DestWorker(OtelDestWorker *s); - virtual ~DestWorker() {}; - static LogThreadedDestWorker *construct(LogThreadedDestDriver *o, gint worker_index); + DestWorker(GrpcDestWorker *s); - virtual bool init(); - virtual void deinit(); - virtual bool connect(); - virtual void disconnect(); - virtual LogThreadedResult insert(LogMessage *msg); - virtual LogThreadedResult flush(LogThreadedFlushMode mode); + LogThreadedResult insert(LogMessage *msg); + LogThreadedResult flush(LogThreadedFlushMode mode); protected: - void prepare_context(::grpc::ClientContext &context); - void clear_current_msg_metadata(); void get_metadata_for_current_msg(LogMessage *msg); @@ -93,9 +81,6 @@ class DestWorker LogThreadedResult flush_spans(); protected: - OtelDestWorker *super; - DestDriver &owner; - std::shared_ptr<::grpc::Channel> channel; std::unique_ptr logs_service_stub; std::unique_ptr metrics_service_stub; @@ -128,12 +113,4 @@ class DestWorker } } -struct OtelDestWorker_ -{ - LogThreadedDestWorker super; - syslogng::grpc::otel::DestWorker *cpp; -}; - -void otel_dw_init_super(LogThreadedDestWorker *s, LogThreadedDestDriver *o, gint worker_index); - #endif diff --git a/modules/grpc/otel/otel-dest.cpp b/modules/grpc/otel/otel-dest.cpp index c7b3406180..8977f44146 100644 --- a/modules/grpc/otel/otel-dest.cpp +++ b/modules/grpc/otel/otel-dest.cpp @@ -24,73 +24,10 @@ #include "otel-dest.hpp" #include "otel-dest-worker.hpp" -#define get_DestDriver(s) (((OtelDestDriver *) s)->cpp) - using namespace syslogng::grpc::otel; /* C++ Implementations */ -DestDriver::DestDriver(OtelDestDriver *s) - : super(s), compression(false), batch_bytes(4 * 1000 * 1000) -{ - credentials_builder_wrapper.self = &credentials_builder; -} - -void -DestDriver::set_url(const char *url_) -{ - url.assign(url_); -} - -const std::string & -DestDriver::get_url() const -{ - return url; -} - -void -DestDriver::set_compression(bool compression_) -{ - compression = compression_; -} - -bool -DestDriver::get_compression() const -{ - return compression; -} - -void -DestDriver::set_batch_bytes(size_t batch_bytes_) -{ - batch_bytes = batch_bytes_; -} - -size_t -DestDriver::get_batch_bytes() const -{ - return batch_bytes; -} - -void -DestDriver::add_extra_channel_arg(std::string name, long value) -{ - int_extra_channel_args.push_back(std::pair {name, value}); -} - -void -DestDriver::add_extra_channel_arg(std::string name, std::string value) -{ - string_extra_channel_args.push_back(std::pair {name, value}); -} - -void -DestDriver::add_header(std::string name, std::string value) -{ - std::transform(name.begin(), name.end(), name.begin(), ::tolower); - headers.push_back(std::pair {name, value}); -} - const char * DestDriver::generate_persist_name() { @@ -118,153 +55,15 @@ DestDriver::format_stats_key(StatsClusterKeyBuilder *kb) LogThreadedDestWorker * DestDriver::construct_worker(int worker_index) { - return DestWorker::construct(&super->super, worker_index); -} - -bool -DestDriver::init() -{ - if (url.length() == 0) - { - msg_error("OpenTelemetry: url() option is mandatory", - log_pipe_location_tag(&super->super.super.super.super)); - return false; - } - - if (!credentials_builder.validate()) - { - return false; - } - - if (!log_threaded_dest_driver_init_method(&this->super->super.super.super.super)) - return false; - - log_threaded_dest_driver_register_aggregated_stats(&this->super->super); - - StatsClusterKeyBuilder *kb = stats_cluster_key_builder_new(); - format_stats_key(kb); - metrics.init(kb, log_pipe_is_internal(&super->super.super.super.super) ? STATS_LEVEL3 : STATS_LEVEL1); - - return true; -} - -bool -DestDriver::deinit() -{ - metrics.deinit(); - return log_threaded_dest_driver_deinit_method(&super->super.super.super.super); -} - -GrpcClientCredentialsBuilderW * -DestDriver::get_credentials_builder_wrapper() -{ - return &credentials_builder_wrapper; -} - -/* C Wrappers */ - -void -otel_dd_set_url(LogDriver *s, const gchar *url) -{ - get_DestDriver(s)->set_url(url); -} - -void -otel_dd_set_compression(LogDriver *s, gboolean enable) -{ - get_DestDriver(s)->set_compression(enable); -} - -void -otel_dd_set_batch_bytes(LogDriver *s, glong b) -{ - get_DestDriver(s)->set_batch_bytes((size_t) b); -} - -void -otel_dd_add_int_channel_arg(LogDriver *s, const gchar *name, glong value) -{ - get_DestDriver(s)->add_extra_channel_arg(name, value); -} - -void -otel_dd_add_string_channel_arg(LogDriver *s, const gchar *name, const gchar *value) -{ - get_DestDriver(s)->add_extra_channel_arg(name, value); -} - -void -otel_dd_add_header(LogDriver *s, const gchar *name, const gchar *value) -{ - get_DestDriver(s)->add_header(name, value); -} - -GrpcClientCredentialsBuilderW * -otel_dd_get_credentials_builder(LogDriver *s) -{ - return get_DestDriver(s)->get_credentials_builder_wrapper(); -} - -static const gchar * -_format_stats_key(LogThreadedDestDriver *s, StatsClusterKeyBuilder *kb) -{ - return get_DestDriver(s)->format_stats_key(kb); -} - -static const gchar * -_generate_persist_name(const LogPipe *s) -{ - return get_DestDriver(s)->generate_persist_name(); -} - -static gboolean -_init(LogPipe *s) -{ - return get_DestDriver(s)->init(); -} - -static gboolean -_deinit(LogPipe *s) -{ - return get_DestDriver(s)->deinit(); -} - -static LogThreadedDestWorker * -_construct_worker(LogThreadedDestDriver *s, gint worker_index) -{ - return get_DestDriver(s)->construct_worker(worker_index); -} - -static void -_free(LogPipe *s) -{ - delete get_DestDriver(s); - log_threaded_dest_driver_free(s); -} - -void -otel_dd_init_super(LogThreadedDestDriver *s, GlobalConfig *cfg) -{ - log_threaded_dest_driver_init_instance(s, cfg); - - s->super.super.super.init = _init; - s->super.super.super.deinit = _deinit; - s->super.super.super.free_fn = _free; - s->super.super.super.generate_persist_name = _generate_persist_name; - - s->worker.construct = _construct_worker; - s->stats_source = stats_register_type("opentelemetry"); - s->format_stats_key = _format_stats_key; - s->metrics.raw_bytes_enabled = TRUE; + GrpcDestWorker *worker = grpc_dw_new(this->super, worker_index); + worker->cpp = new DestWorker(worker); + return &worker->super; } LogDriver * otel_dd_new(GlobalConfig *cfg) { - OtelDestDriver *self = g_new0(OtelDestDriver, 1); - - otel_dd_init_super(&self->super, cfg); + GrpcDestDriver *self = grpc_dd_new(cfg, "opentelemetry"); self->cpp = new DestDriver(self); - return &self->super.super.super; } diff --git a/modules/grpc/otel/otel-dest.h b/modules/grpc/otel/otel-dest.h index dd605073a7..4f79e79928 100644 --- a/modules/grpc/otel/otel-dest.h +++ b/modules/grpc/otel/otel-dest.h @@ -34,13 +34,6 @@ typedef struct OtelDestDriver_ OtelDestDriver; LogDriver *otel_dd_new(GlobalConfig *cfg); -void otel_dd_set_url(LogDriver *s, const gchar *url); -void otel_dd_set_compression(LogDriver *s, gboolean enable); -void otel_dd_set_batch_bytes(LogDriver *s, glong b); -void otel_dd_add_int_channel_arg(LogDriver *s, const gchar *name, glong value); -void otel_dd_add_string_channel_arg(LogDriver *s, const gchar *name, const gchar *value); -void otel_dd_add_header(LogDriver *s, const gchar *name, const gchar *value); -GrpcClientCredentialsBuilderW *otel_dd_get_credentials_builder(LogDriver *s); #include "compat/cpp-end.h" diff --git a/modules/grpc/otel/otel-dest.hpp b/modules/grpc/otel/otel-dest.hpp index ce76efebe8..4c26ae9deb 100644 --- a/modules/grpc/otel/otel-dest.hpp +++ b/modules/grpc/otel/otel-dest.hpp @@ -30,8 +30,7 @@ #include "logthrdest/logthrdestdrv.h" #include "compat/cpp-end.h" -#include "credentials/grpc-credentials-builder.hpp" -#include "metrics/grpc-metrics.hpp" +#include "grpc-dest.hpp" #include @@ -41,60 +40,21 @@ namespace syslogng { namespace grpc { namespace otel { -class DestDriver +class DestDriver: public syslogng::grpc::DestDriver { public: - DestDriver(OtelDestDriver *s); - virtual ~DestDriver() {}; + DestDriver(GrpcDestDriver *s) : syslogng::grpc::DestDriver(s) {}; - void set_url(const char *url); - const std::string &get_url() const; - - void set_compression(bool enable); - bool get_compression() const; - - void set_batch_bytes(size_t bytes); - size_t get_batch_bytes() const; - - void add_extra_channel_arg(std::string name, long value); - void add_extra_channel_arg(std::string name, std::string value); - - void add_header(std::string name, std::string value); - - virtual bool init(); - virtual bool deinit(); - virtual const char *format_stats_key(StatsClusterKeyBuilder *kb); - virtual const char *generate_persist_name(); - virtual LogThreadedDestWorker *construct_worker(int worker_index); - - GrpcClientCredentialsBuilderW *get_credentials_builder_wrapper(); - -public: - syslogng::grpc::ClientCredentialsBuilder credentials_builder; + const char *format_stats_key(StatsClusterKeyBuilder *kb); + const char *generate_persist_name(); + LogThreadedDestWorker *construct_worker(int worker_index); protected: friend class DestWorker; - OtelDestDriver *super; - std::string url; - bool compression; - size_t batch_bytes; - std::list> int_extra_channel_args; - std::list> string_extra_channel_args; - std::list> headers; - GrpcClientCredentialsBuilderW credentials_builder_wrapper; - DestDriverMetrics metrics; }; } } } -struct OtelDestDriver_ -{ - LogThreadedDestDriver super; - syslogng::grpc::otel::DestDriver *cpp; -}; - -void otel_dd_init_super(LogThreadedDestDriver *s, GlobalConfig *cfg); - #endif diff --git a/modules/grpc/otel/otel-grammar.ym b/modules/grpc/otel/otel-grammar.ym index 02c5103ffc..5c5ced9c3e 100644 --- a/modules/grpc/otel/otel-grammar.ym +++ b/modules/grpc/otel/otel-grammar.ym @@ -35,6 +35,7 @@ #include "otel-protobuf-parser.h" #include "otel-dest.h" #include "syslog-ng-otlp-dest.h" +#include "grpc-dest.h" GrpcServerCredentialsBuilderW *last_grpc_server_credentials_builder; GrpcClientCredentialsBuilderW *last_grpc_client_credentials_builder; @@ -163,10 +164,10 @@ destination_otel_options ; destination_otel_option - : KW_URL '(' string ')' { otel_dd_set_url(last_driver, $3); free($3); } - | KW_AUTH { last_grpc_client_credentials_builder = otel_dd_get_credentials_builder(last_driver); } '(' grpc_client_credentials_option ')' - | KW_COMPRESSION '(' yesno ')' { otel_dd_set_compression(last_driver, $3); } - | KW_BATCH_BYTES '(' positive_integer ')' { otel_dd_set_batch_bytes(last_driver, $3); } + : KW_URL '(' string ')' { grpc_dd_set_url(last_driver, $3); free($3); } + | KW_AUTH { last_grpc_client_credentials_builder = grpc_dd_get_credentials_builder(last_driver); } '(' grpc_client_credentials_option ')' + | KW_COMPRESSION '(' yesno ')' { grpc_dd_set_compression(last_driver, $3); } + | KW_BATCH_BYTES '(' positive_integer ')' { grpc_dd_set_batch_bytes(last_driver, $3); } | KW_CHANNEL_ARGS '(' destination_otel_channel_args ')' | KW_HEADERS '(' destination_otel_headers ')' | threaded_dest_driver_general_option @@ -180,8 +181,8 @@ destination_otel_channel_args ; destination_otel_channel_arg - : string LL_ARROW LL_NUMBER { otel_dd_add_int_channel_arg(last_driver, $1, $3); free($1); } - | string LL_ARROW string { otel_dd_add_string_channel_arg(last_driver, $1, $3); free($1); free($3); } + : string LL_ARROW LL_NUMBER { grpc_dd_add_int_channel_arg(last_driver, $1, $3); free($1); } + | string LL_ARROW string { grpc_dd_add_string_channel_arg(last_driver, $1, $3); free($1); free($3); } ; destination_otel_headers @@ -190,7 +191,7 @@ destination_otel_headers ; destination_otel_header - : string LL_ARROW string { otel_dd_add_header(last_driver, $1, $3); free($1); free($3); } + : string LL_ARROW string { grpc_dd_add_header(last_driver, $1, $3); free($1); free($3); } ; destination_syslog_ng_otlp diff --git a/modules/grpc/otel/syslog-ng-otlp-dest-worker.cpp b/modules/grpc/otel/syslog-ng-otlp-dest-worker.cpp index a841759154..b6f479ab2f 100644 --- a/modules/grpc/otel/syslog-ng-otlp-dest-worker.cpp +++ b/modules/grpc/otel/syslog-ng-otlp-dest-worker.cpp @@ -25,17 +25,6 @@ using namespace syslogng::grpc::otel; using namespace opentelemetry::proto::logs::v1; -LogThreadedDestWorker * -SyslogNgDestWorker::construct(LogThreadedDestDriver *o, gint worker_index) -{ - SyslogNgOtlpDestWorker *self = g_new0(SyslogNgOtlpDestWorker, 1); - - otel_dw_init_super(&self->super, o, worker_index); - self->cpp = new SyslogNgDestWorker(self); - - return &self->super; -} - ScopeLogs * SyslogNgDestWorker::lookup_scope_logs(LogMessage *msg) { diff --git a/modules/grpc/otel/syslog-ng-otlp-dest-worker.hpp b/modules/grpc/otel/syslog-ng-otlp-dest-worker.hpp index b6d6e54d5b..db9b8ad7e8 100644 --- a/modules/grpc/otel/syslog-ng-otlp-dest-worker.hpp +++ b/modules/grpc/otel/syslog-ng-otlp-dest-worker.hpp @@ -26,8 +26,6 @@ #include "syslog-ng-otlp-dest.hpp" #include "otel-dest-worker.hpp" -typedef OtelDestWorker SyslogNgOtlpDestWorker; - namespace syslogng { namespace grpc { namespace otel { @@ -36,7 +34,6 @@ class SyslogNgDestWorker : public DestWorker { public: using DestWorker::DestWorker; - static LogThreadedDestWorker *construct(LogThreadedDestDriver *o, gint worker_index); ScopeLogs *lookup_scope_logs(LogMessage *msg) override; LogThreadedResult insert(LogMessage *msg) override; diff --git a/modules/grpc/otel/syslog-ng-otlp-dest.cpp b/modules/grpc/otel/syslog-ng-otlp-dest.cpp index ecd02375af..9bb49524e2 100644 --- a/modules/grpc/otel/syslog-ng-otlp-dest.cpp +++ b/modules/grpc/otel/syslog-ng-otlp-dest.cpp @@ -68,7 +68,9 @@ SyslogNgDestDriver::format_stats_key(StatsClusterKeyBuilder *kb) LogThreadedDestWorker * SyslogNgDestDriver::construct_worker(int worker_index) { - return SyslogNgDestWorker::construct(&super->super, worker_index); + GrpcDestWorker *worker = grpc_dw_new(this->super, worker_index); + worker->cpp = new SyslogNgDestWorker(worker); + return &worker->super; } bool @@ -107,11 +109,7 @@ SyslogNgDestDriver::init() LogDriver * syslog_ng_otlp_dd_new(GlobalConfig *cfg) { - SyslogNgOtlpDestDriverWrapper *self = g_new0(SyslogNgOtlpDestDriverWrapper, 1); - - otel_dd_init_super(&self->super, cfg); - self->super.stats_source = stats_register_type("syslog-ng-otlp"); + GrpcDestDriver *self = grpc_dd_new(cfg, "syslog-ng-otlp"); self->cpp = new SyslogNgDestDriver(self); - return &self->super.super.super; } diff --git a/modules/grpc/otel/syslog-ng-otlp-dest.h b/modules/grpc/otel/syslog-ng-otlp-dest.h index e00816325d..364214be91 100644 --- a/modules/grpc/otel/syslog-ng-otlp-dest.h +++ b/modules/grpc/otel/syslog-ng-otlp-dest.h @@ -29,8 +29,6 @@ #include "otel-dest.h" -typedef OtelDestDriver SyslogNgOtlpDestDriverWrapper; - LogDriver *syslog_ng_otlp_dd_new(GlobalConfig *cfg); #include "compat/cpp-end.h" From 727e67581f15aaf3e19875e928242dd9af80ad37 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Tue, 3 Sep 2024 21:49:04 +0200 Subject: [PATCH 04/25] loki: derive DestinationDriver and DestinationWorker from grpc base classes Signed-off-by: Attila Szakacs --- modules/grpc/loki/CMakeLists.txt | 1 - modules/grpc/loki/Makefile.am | 2 - modules/grpc/loki/loki-dest.cpp | 185 +++++------------------------- modules/grpc/loki/loki-dest.h | 14 --- modules/grpc/loki/loki-dest.hpp | 72 +----------- modules/grpc/loki/loki-grammar.ym | 17 +-- modules/grpc/loki/loki-worker.cpp | 151 ++++-------------------- modules/grpc/loki/loki-worker.h | 37 ------ modules/grpc/loki/loki-worker.hpp | 13 +-- 9 files changed, 72 insertions(+), 420 deletions(-) delete mode 100644 modules/grpc/loki/loki-worker.h diff --git a/modules/grpc/loki/CMakeLists.txt b/modules/grpc/loki/CMakeLists.txt index 7136edf142..d31d3c84f4 100644 --- a/modules/grpc/loki/CMakeLists.txt +++ b/modules/grpc/loki/CMakeLists.txt @@ -10,7 +10,6 @@ set(LOKI_CPP_SOURCES loki-dest.h loki-worker.hpp loki-worker.cpp - loki-worker.h ) set(LOKI_SOURCES diff --git a/modules/grpc/loki/Makefile.am b/modules/grpc/loki/Makefile.am index 37aefd703c..c374797e7c 100644 --- a/modules/grpc/loki/Makefile.am +++ b/modules/grpc/loki/Makefile.am @@ -6,7 +6,6 @@ modules_grpc_loki_libloki_cpp_la_SOURCES = \ modules/grpc/loki/loki-dest.h \ modules/grpc/loki/loki-dest.hpp \ modules/grpc/loki/loki-dest.cpp \ - modules/grpc/loki/loki-worker.h \ modules/grpc/loki/loki-worker.hpp \ modules/grpc/loki/loki-worker.cpp @@ -35,7 +34,6 @@ modules_grpc_loki_libloki_la_SOURCES = \ modules_grpc_loki_libloki_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ $(GRPC_COMMON_CFLAGS) \ - -I$(top_srcdir)/modules/grpc \ -I$(top_srcdir)/modules/grpc/loki \ -I$(top_builddir)/modules/grpc/loki \ -I$(top_srcdir)/modules/grpc diff --git a/modules/grpc/loki/loki-dest.cpp b/modules/grpc/loki/loki-dest.cpp index 1eacfe2d71..ac6dff79aa 100644 --- a/modules/grpc/loki/loki-dest.cpp +++ b/modules/grpc/loki/loki-dest.cpp @@ -39,18 +39,11 @@ constexpr const auto DEFAULT_MESSAGE_TEMPLATE = "$ISODATE $HOST $MSGHDR$MSG"; using syslogng::grpc::loki::DestinationDriver; -struct _LokiDestDriver -{ - LogThreadedDestDriver super; - DestinationDriver *cpp; -}; - -DestinationDriver::DestinationDriver(LokiDestDriver *s) - : super(s), url("localhost:9095"), timestamp(LM_TS_PROCESSED), - keepalive_time(-1), keepalive_timeout(-1), keepalive_max_pings_without_data(-1) +DestinationDriver::DestinationDriver(GrpcDestDriver *s) + : syslogng::grpc::DestDriver(s), timestamp(LM_TS_PROCESSED) { + this->url = "localhost:9095"; log_template_options_defaults(&this->template_options); - credentials_builder_wrapper.self = &credentials_builder; } DestinationDriver::~DestinationDriver() @@ -70,11 +63,6 @@ DestinationDriver::init() { GlobalConfig *cfg = log_pipe_get_config(&this->super->super.super.super.super); - if (!credentials_builder.validate()) - { - return false; - } - if (!this->message) { this->message = log_template_new(cfg, NULL); @@ -109,25 +97,11 @@ DestinationDriver::init() else log_threaded_dest_driver_set_worker_partition_key_ref(&this->super->super.super.super, worker_partition_key); - if (!log_threaded_dest_driver_init_method(&this->super->super.super.super.super)) - return false; - - StatsClusterKeyBuilder *kb = stats_cluster_key_builder_new(); - this->format_stats_key(kb); - this->metrics.init(kb, log_pipe_is_internal(&this->super->super.super.super.super) ? STATS_LEVEL3 : STATS_LEVEL1); - - return true; -} - -bool -DestinationDriver::deinit() -{ - this->metrics.deinit(); - return log_threaded_dest_driver_deinit_method(&this->super->super.super.super.super); + return syslogng::grpc::DestDriver::init(); } const gchar * -DestinationDriver::format_persist_name() +DestinationDriver::generate_persist_name() { static gchar persist_name[1024]; @@ -149,169 +123,66 @@ DestinationDriver::format_stats_key(StatsClusterKeyBuilder *kb) return nullptr; } -GrpcClientCredentialsBuilderW * -DestinationDriver::get_credentials_builder_wrapper() +LogThreadedDestWorker * +DestinationDriver::construct_worker(int worker_index) { - return &this->credentials_builder_wrapper; + GrpcDestWorker *worker = grpc_dw_new(this->super, worker_index); + worker->cpp = new DestinationWorker(worker); + return &worker->super; } - /* C Wrappers */ DestinationDriver * -loki_dd_get_cpp(LokiDestDriver *self) -{ - return self->cpp; -} - -static const gchar * -_format_persist_name(const LogPipe *s) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - return self->cpp->format_persist_name(); -} - -static const gchar * -_format_stats_key(LogThreadedDestDriver *s, StatsClusterKeyBuilder *kb) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - return self->cpp->format_stats_key(kb); -} - -GrpcClientCredentialsBuilderW * -loki_dd_get_credentials_builder(LogDriver *s) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - return self->cpp->get_credentials_builder_wrapper(); -} - -void -loki_dd_set_url(LogDriver *d, const gchar *url) +loki_dd_get_cpp(GrpcDestDriver *self) { - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->set_url(url); + return (DestinationDriver *) self->cpp; } void loki_dd_set_message_template_ref(LogDriver *d, LogTemplate *message) { - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->set_message_template_ref(message); + GrpcDestDriver *self = (GrpcDestDriver *) d; + DestinationDriver *cpp = loki_dd_get_cpp(self); + cpp->set_message_template_ref(message); } void loki_dd_add_label(LogDriver *d, const gchar *name, LogTemplate *value) { - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->add_label(name, value); + GrpcDestDriver *self = (GrpcDestDriver *) d; + DestinationDriver *cpp = loki_dd_get_cpp(self); + cpp->add_label(name, value); } gboolean loki_dd_set_timestamp(LogDriver *d, const gchar *t) { - LokiDestDriver *self = (LokiDestDriver *) d; - return self->cpp->set_timestamp(t); + GrpcDestDriver *self = (GrpcDestDriver *) d; + DestinationDriver *cpp = loki_dd_get_cpp(self); + return cpp->set_timestamp(t); } void loki_dd_set_tenant_id(LogDriver *d, const gchar *tid) { - LokiDestDriver *self = (LokiDestDriver *) d; - return self->cpp->set_tenant_id(tid); -} - -void -loki_dd_set_keepalive_time(LogDriver *d, gint t) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->set_keepalive_time(t); -} - -void -loki_dd_set_keepalive_timeout(LogDriver *d, gint t) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->set_keepalive_timeout(t); -} - -void -loki_dd_set_keepalive_max_pings(LogDriver *d, gint p) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->set_keepalive_max_pings(p); -} - -void -loki_dd_add_int_channel_arg(LogDriver *d, const gchar *name, glong value) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->add_extra_channel_arg(name, value); -} - -void -loki_dd_add_string_channel_arg(LogDriver *d, const gchar *name, const gchar *value) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->add_extra_channel_arg(name, value); -} - -void -loki_dd_add_header(LogDriver *d, const gchar *name, const gchar *value) -{ - LokiDestDriver *self = (LokiDestDriver *) d; - self->cpp->add_header(name, value); + GrpcDestDriver *self = (GrpcDestDriver *) d; + DestinationDriver *cpp = loki_dd_get_cpp(self); + return cpp->set_tenant_id(tid); } LogTemplateOptions * loki_dd_get_template_options(LogDriver *d) { - LokiDestDriver *self = (LokiDestDriver *) d; - return &self->cpp->get_template_options(); -} - -static gboolean -_init(LogPipe *s) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - return self->cpp->init(); -} - -static gboolean -_deinit(LogPipe *s) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - return self->cpp->deinit(); -} - -static void -_free(LogPipe *s) -{ - LokiDestDriver *self = (LokiDestDriver *) s; - delete self->cpp; - - log_threaded_dest_driver_free(s); + GrpcDestDriver *self = (GrpcDestDriver *) d; + DestinationDriver *cpp = loki_dd_get_cpp(self); + return &cpp->get_template_options(); } LogDriver * loki_dd_new(GlobalConfig *cfg) { - LokiDestDriver *self = g_new0(LokiDestDriver, 1); - - log_threaded_dest_driver_init_instance(&self->super, cfg); - + GrpcDestDriver *self = grpc_dd_new(cfg, "loki"); self->cpp = new DestinationDriver(self); - - self->super.super.super.super.init = _init; - self->super.super.super.super.deinit = _deinit; - self->super.super.super.super.free_fn = _free; - self->super.super.super.super.generate_persist_name = _format_persist_name; - - self->super.format_stats_key = _format_stats_key; - self->super.stats_source = stats_register_type("loki"); - - self->super.worker.construct = loki_dw_new; - - self->super.flush_on_key_change = TRUE; - return &self->super.super.super; } diff --git a/modules/grpc/loki/loki-dest.h b/modules/grpc/loki/loki-dest.h index af2ee4a13a..2c9773e097 100644 --- a/modules/grpc/loki/loki-dest.h +++ b/modules/grpc/loki/loki-dest.h @@ -33,27 +33,13 @@ #include "template/templates.h" #include "credentials/grpc-credentials-builder.h" -typedef struct _LokiDestDriver LokiDestDriver; - LogDriver *loki_dd_new(GlobalConfig *cfg); -void loki_dd_set_url(LogDriver *d, const gchar *url); void loki_dd_set_message_template_ref(LogDriver *d, LogTemplate *message); void loki_dd_add_label(LogDriver *d, const gchar *name, LogTemplate *value); gboolean loki_dd_set_timestamp(LogDriver *d, const gchar *t); void loki_dd_set_tenant_id(LogDriver *d, const gchar *tid); -GrpcClientCredentialsBuilderW *loki_dd_get_credentials_builder(LogDriver *s); - -void loki_dd_set_keepalive_time(LogDriver *d, gint t); -void loki_dd_set_keepalive_timeout(LogDriver *d, gint t); -void loki_dd_set_keepalive_max_pings(LogDriver *d, gint p); - -void loki_dd_add_int_channel_arg(LogDriver *s, const gchar *name, glong value); -void loki_dd_add_string_channel_arg(LogDriver *s, const gchar *name, const gchar *value); - -void loki_dd_add_header(LogDriver *s, const gchar *name, const gchar *value); - LogTemplateOptions *loki_dd_get_template_options(LogDriver *d); #include "compat/cpp-end.h" diff --git a/modules/grpc/loki/loki-dest.hpp b/modules/grpc/loki/loki-dest.hpp index 54ed37b4a1..8b4f70d449 100644 --- a/modules/grpc/loki/loki-dest.hpp +++ b/modules/grpc/loki/loki-dest.hpp @@ -33,8 +33,7 @@ #include "logmsg/logmsg.h" #include "compat/cpp-end.h" -#include "credentials/grpc-credentials-builder.hpp" -#include "metrics/grpc-metrics.hpp" +#include "grpc-dest.hpp" #include #include @@ -71,16 +70,15 @@ struct Label }; -class DestinationDriver final +class DestinationDriver final : public syslogng::grpc::DestDriver { public: - DestinationDriver(LokiDestDriver *s); + DestinationDriver(GrpcDestDriver *s); ~DestinationDriver(); bool init(); - bool deinit(); - const gchar *format_persist_name(); + const gchar *generate_persist_name(); const gchar *format_stats_key(StatsClusterKeyBuilder *kb); - GrpcClientCredentialsBuilderW *get_credentials_builder_wrapper(); + LogThreadedDestWorker *construct_worker(int worker_index); void add_label(std::string name, LogTemplate *value); @@ -89,11 +87,6 @@ class DestinationDriver final return this->template_options; } - void set_url(std::string u) - { - this->url = u; - } - void set_message_template_ref(LogTemplate *msg) { log_template_unref(this->message); @@ -113,73 +106,20 @@ class DestinationDriver final return true; } - void set_keepalive_time(int t) - { - this->keepalive_time = t; - } - - void set_keepalive_timeout(int t) - { - this->keepalive_timeout = t; - } - - void set_keepalive_max_pings(int p) - { - this->keepalive_max_pings_without_data = p; - } - void set_tenant_id(std::string tid) { this->tenant_id = tid; } - void add_extra_channel_arg(std::string name, long value) - { - this->int_extra_channel_args.push_back(std::pair {name, value}); - } - - void add_extra_channel_arg(std::string name, std::string value) - { - this->string_extra_channel_args.push_back(std::pair {name, value}); - } - - void add_header(std::string name, std::string value) - { - std::transform(name.begin(), name.end(), name.begin(), ::tolower); - this->headers.push_back(std::pair {name, value}); - } - - const std::string &get_url() - { - return this->url; - } - private: friend class DestinationWorker; - -private: - LokiDestDriver *super; LogTemplateOptions template_options; - std::string url; std::string tenant_id; LogTemplate *message = nullptr; std::vector