From c90faa1e0345e3796fa56af1c38afe1ea22318b3 Mon Sep 17 00:00:00 2001 From: Chris Larsen Date: Fri, 23 Feb 2024 15:46:52 -0700 Subject: [PATCH] feat: add vmware log insight fluentd output * add fluentd output plugin for VMware LogInsight --- ...logging.banzaicloud.io_clusteroutputs.yaml | 454 ++++++++++++++++++ .../crds/logging.banzaicloud.io_outputs.yaml | 227 +++++++++ ...logging.banzaicloud.io_clusteroutputs.yaml | 454 ++++++++++++++++++ .../bases/logging.banzaicloud.io_outputs.yaml | 227 +++++++++ .../crds/v1beta1/output_types.md | 3 + docs/configuration/plugins/_index.md | 1 + .../plugins/outputs/vmware_loginsight.md | 173 +++++++ pkg/sdk/logging/api/v1beta1/output_types.go | 4 +- .../api/v1beta1/zz_generated.deepcopy.go | 5 + .../logging/model/output/vmware_loginsight.go | 188 ++++++++ .../model/output/vmware_loginsight_test.go | 101 ++++ .../model/output/zz_generated.deepcopy.go | 69 +++ 12 files changed, 1905 insertions(+), 1 deletion(-) create mode 100644 docs/configuration/plugins/outputs/vmware_loginsight.md create mode 100644 pkg/sdk/logging/model/output/vmware_loginsight.go create mode 100644 pkg/sdk/logging/model/output/vmware_loginsight_test.go diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml index 16f31f2d5c..a480003883 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml @@ -6557,6 +6557,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: @@ -13115,6 +13342,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml index dccbcf59c8..26ce7f9a56 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml @@ -12793,6 +12793,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml index 16f31f2d5c..a480003883 100644 --- a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml @@ -6557,6 +6557,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: @@ -13115,6 +13342,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml index dccbcf59c8..26ce7f9a56 100644 --- a/config/crd/bases/logging.banzaicloud.io_outputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_outputs.yaml @@ -12793,6 +12793,233 @@ spec: required: - host type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object type: object status: properties: diff --git a/docs/configuration/crds/v1beta1/output_types.md b/docs/configuration/crds/v1beta1/output_types.md index 40b55136bb..493c0934a9 100644 --- a/docs/configuration/crds/v1beta1/output_types.md +++ b/docs/configuration/crds/v1beta1/output_types.md @@ -95,6 +95,9 @@ OutputSpec defines the desired state of Output ### syslog (*output.SyslogOutputConfig, optional) {#outputspec-syslog} +### vmwareLogInsight (*output.VMwareLogInsightOutput, optional) {#outputspec-vmwareloginsight} + + ## OutputStatus diff --git a/docs/configuration/plugins/_index.md b/docs/configuration/plugins/_index.md index 1b4f1c9190..af0c93cfb1 100644 --- a/docs/configuration/plugins/_index.md +++ b/docs/configuration/plugins/_index.md @@ -57,6 +57,7 @@ For more information please click on the plugin name | **[SQS](outputs/sqs/)** | outputs | Output plugin writes fluent-events as queue messages to Amazon SQS | Testing | [v2.1.0](https://github.com/ixixi/fluent-plugin-sqs) | | **[SumoLogic](outputs/sumologic/)** | outputs | Send your logs to Sumologic | GA | [1.8.0](https://github.com/SumoLogic/fluentd-output-sumologic/releases/tag/1.8.0) | | **[Syslog](outputs/syslog/)** | outputs | Output plugin writes events to syslog | GA | [0.9.0.rc.8](https://github.com/cloudfoundry/fluent-plugin-syslog_rfc5424) | +| **[VMware LogInsight](outputs/vmware_loginsight/)** | outputs | Store logs in VMware LogInsight | GA | [1.4.2](https://github.com/vmware/fluent-plugin-vmware-loginsight/releases/tag/v1.4.2) | | **[Syslog-NG Match](syslogng-filters/match/)** | syslogng-filters | Selectively keep records | GA | [more info](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/65#TOPIC-1829159) | | **[Syslog-NG Parser](syslogng-filters/parser/)** | syslogng-filters | Parse data from records | GA | [more info](https://axoflow.com/docs/axosyslog-core/chapter-parsers/) | | **[Syslog-NG Rewrite](syslogng-filters/rewrite/)** | syslogng-filters | Rewrite parts of the message | GA | [more info](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/77) | diff --git a/docs/configuration/plugins/outputs/vmware_loginsight.md b/docs/configuration/plugins/outputs/vmware_loginsight.md new file mode 100644 index 0000000000..dd31575423 --- /dev/null +++ b/docs/configuration/plugins/outputs/vmware_loginsight.md @@ -0,0 +1,173 @@ +--- +title: VMware LogInsight +weight: 200 +generated_file: true +--- + +# VMware LogInsight output plugin for Fluentd +## Overview + +For details, see [https://github.com/vmware/fluent-plugin-vmware-loginsight](https://github.com/vmware/fluent-plugin-vmware-loginsight). + +## Example output configurations + +```yaml +spec: + vmwareLogInsight: + scheme: https + ssl_verify: true + host: MY_LOGINSIGHT_HOST + port: 9543 + agent_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + log_text_keys: + - log + - msg + - message + http_conn_debug: false +``` + + +## Configuration +## VMwareLogInsight + +LogZ Send your logs to VNMware LogInsight + +### agent_id (string, optional) {#vmwareloginsight-agent_id} + +agent_id generated by your LI + +Default: 0 + +### authentication (*string, optional) {#vmwareloginsight-authentication} + +Type of authentication to use (nil,basic) + +Default: nil + +### buffer (*Buffer, optional) {#vmwareloginsight-buffer} + +[Buffer](../buffer/) + + +### ca_file (*secret.Secret, optional) {#vmwareloginsight-ca_file} + +[Secret](../secret/) + + +### config_param (map[string]string, optional) {#vmwareloginsight-config_param} + +Rename fields names + +Default: {"source" => "log_source"} + +### flatten_hashes (*bool, optional) {#vmwareloginsight-flatten_hashes} + +Flatten hashes to create one key/val pair w/o losing log data + +Default: true + +### flatten_hashes_separator (string, optional) {#vmwareloginsight-flatten_hashes_separator} + +Separator to use for joining flattened keys + +Default: _ + +### http_conn_debug (bool, optional) {#vmwareloginsight-http_conn_debug} + +If set, enables debug logs for http connection + +Default: false + +### http_method (string, optional) {#vmwareloginsight-http_method} + +HTTP method (post) + +Default: post + +### host (string, optional) {#vmwareloginsight-host} + +VMware Aria Operations For Logs Host ex. localhost + + +### log_text_keys ([]string, optional) {#vmwareloginsight-log_text_keys} + +Keys from log event whose values should be added as log message/text to VMware Aria Operations For Logs. These key/value pairs won't be expanded/flattened and won't be added as metadata/fields. + +Default: ["log", "message", "msg"] + +### max_batch_size (int, optional) {#vmwareloginsight-max_batch_size} + +Number of bytes per post request + +Default: 4000000 + +### password (*secret.Secret, optional) {#vmwareloginsight-password} + +[Secret](../secret/) + + +### path (string, optional) {#vmwareloginsight-path} + +VMware Aria Operations For Logs ingestion api path ex. 'api/v1/events/ingest' + +Default: api/v1/events/ingest + +### port (int, optional) {#vmwareloginsight-port} + +VMware Aria Operations For Logs port ex. 9000 + +Default: 80 + +### raise_on_error (bool, optional) {#vmwareloginsight-raise_on_error} + +Raise errors that were rescued during HTTP requests? + +Default: false + +### rate_limit_msec (int, optional) {#vmwareloginsight-rate_limit_msec} + +Simple rate limiting: ignore any records within `rate_limit_msec` since the last one + +Default: 0 + +### request_retries (int, optional) {#vmwareloginsight-request_retries} + +Number of retries + +Default: 3 + +### request_timeout (int, optional) {#vmwareloginsight-request_timeout} + +http connection ttl for each request + +Default: 5 + +### ssl_verify (*bool, optional) {#vmwareloginsight-ssl_verify} + +SSL verification flag + +Default: true + +### scheme (string, optional) {#vmwareloginsight-scheme} + +HTTP scheme (http,https) + +Default: http + +### serializer (string, optional) {#vmwareloginsight-serializer} + +Serialization (json) + +Default: json + +### shorten_keys (map[string]string, optional) {#vmwareloginsight-shorten_keys} + +Keys from log event to rewrite for instance from 'kubernetes_namespace' to 'k8s_namespace' tags will be rewritten with substring substitution and applied in the order present in the hash (Hashes enumerate their values in the order that the corresponding keys were inserted see: https://ruby-doc.org/core-2.2.2/Hash.html (default { 'kubernetes_':'k8s_', 'namespace':'ns', 'labels_':'', '_name':'', '_hash':'', 'container_':'' }) + + +### username (*secret.Secret, optional) {#vmwareloginsight-username} + +[Secret](../secret/) + + + diff --git a/pkg/sdk/logging/api/v1beta1/output_types.go b/pkg/sdk/logging/api/v1beta1/output_types.go index f446b7c34e..f9bbd8bce3 100644 --- a/pkg/sdk/logging/api/v1beta1/output_types.go +++ b/pkg/sdk/logging/api/v1beta1/output_types.go @@ -15,8 +15,9 @@ package v1beta1 import ( - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" ) // +name:"OutputSpec" @@ -59,6 +60,7 @@ type OutputSpec struct { SQSOutputConfig *output.SQSOutputConfig `json:"sqs,omitempty"` MattermostOutputConfig *output.MattermostOutputConfig `json:"mattermost,omitempty"` RelabelOutputConfig *output.RelabelOutputConfig `json:"relabel,omitempty"` + VMwareLogInsightConfig *output.VMwareLogInsightOutput `json:"vmwareLogInsight,omitempty"` } // OutputStatus defines the observed state of Output diff --git a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go index 2e05c58f21..a16b8c5210 100644 --- a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go @@ -2526,6 +2526,11 @@ func (in *OutputSpec) DeepCopyInto(out *OutputSpec) { *out = new(output.RelabelOutputConfig) **out = **in } + if in.VMwareLogInsightConfig != nil { + in, out := &in.VMwareLogInsightConfig, &out.VMwareLogInsightConfig + *out = new(output.VMwareLogInsightOutput) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OutputSpec. diff --git a/pkg/sdk/logging/model/output/vmware_loginsight.go b/pkg/sdk/logging/model/output/vmware_loginsight.go new file mode 100644 index 0000000000..489128137d --- /dev/null +++ b/pkg/sdk/logging/model/output/vmware_loginsight.go @@ -0,0 +1,188 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package output + +import ( + "github.com/cisco-open/operator-tools/pkg/secret" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types" +) + +// +name:"VMware LogInsight" +// +weight:"200" +type _hugoVMwareLogInsight interface{} //nolint:deadcode,unused + +// +docName:"VMware LogInsight output plugin for Fluentd" +/* +For details, see [https://github.com/vmware/fluent-plugin-vmware-loginsight](https://github.com/vmware/fluent-plugin-vmware-loginsight). + +## Example output configurations + +```yaml +spec: + vmwareLogInsight: + scheme: https + ssl_verify: true + host: MY_LOGINSIGHT_HOST + port: 9543 + agent_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + log_text_keys: + - log + - msg + - message + http_conn_debug: false +``` +*/ +type _docVMwareLogInsight interface{} //nolint:deadcode,unused + +// +name:"VMware LogInsight" +// +url:"https://github.com/vmware/fluent-plugin-vmware-loginsight/releases/tag/v1.4.2" +// +version:"1.4.2" +// +description:"Store logs in VMware LogInsight" +// +status:"GA" +type _metaVMwareLogInsight interface{} //nolint:deadcode,unused + +// +kubebuilder:object:generate=true +// +docName:"VMwareLogInsight" +// LogZ Send your logs to VNMware LogInsight +type VMwareLogInsightOutput struct { + // agent_id generated by your LI (default: 0) + AgentID string `json:"agent_id,omitempty"` + + // +kubebuilder:validation:Enum:=nil;basic + + // Type of authentication to use (nil,basic) (default: nil) + Authentication *string `json:"authentication,omitempty"` + + // +docLink:"Buffer,../buffer/" + Buffer *Buffer `json:"buffer,omitempty"` + + // +docLink:"Secret,../secret/" + CAFile *secret.Secret `json:"ca_file,omitempty"` + + // Rename fields names (default: {"source" => "log_source"}) + ConfigParam map[string]string `json:"config_param,omitempty"` + + // Flatten hashes to create one key/val pair w/o losing log data (default: true) + FlattenHashes *bool `json:"flatten_hashes,omitempty"` + + // Separator to use for joining flattened keys (default: _) + FlattenHashesSeparator string `json:"flatten_hashes_separator,omitempty"` + + // If set, enables debug logs for http connection (default: false) + HTTPConnectionDebug bool `json:"http_conn_debug,omitempty"` + + // +kubebuilder:validation:Enum:=post + + // HTTP method (post) (default: post) + HTTPMethod string `json:"http_method,omitempty"` + + // VMware Aria Operations For Logs Host ex. localhost + Host string `json:"host,omitempty"` + + // Keys from log event whose values should be added as log message/text to + // VMware Aria Operations For Logs. These key/value pairs won't be + // expanded/flattened and won't be added as metadata/fields. + // + // (default: ["log", "message", "msg"]) + LogTextKeys []string `json:"log_text_keys,omitempty"` + + // Number of bytes per post request (default: 4000000) + MaxBatchSize int `json:"max_batch_size,omitempty"` + + // +docLink:"Secret,../secret/" + Password *secret.Secret `json:"password,omitempty"` + + // VMware Aria Operations For Logs ingestion api path ex. 'api/v1/events/ingest' + // (default: api/v1/events/ingest) + Path string `json:"path,omitempty"` + + // VMware Aria Operations For Logs port ex. 9000 (default: 80) + Port int `json:"port,omitempty"` + + // Raise errors that were rescued during HTTP requests? (default: false) + RaiseOnError bool `json:"raise_on_error,omitempty"` + + // Simple rate limiting: ignore any records within `rate_limit_msec` since the + // last one (default: 0) + RateLimitMilliseconds int `json:"rate_limit_msec,omitempty"` + + // Number of retries (default: 3) + RequestRetries int `json:"request_retries,omitempty"` + + // http connection ttl for each request (default: 5) + RequestTimeout int `json:"request_timeout,omitempty"` + + // SSL verification flag (default: true) + SSLVerify *bool `json:"ssl_verify,omitempty"` + + // +kubebuilder:validation:Enum:=http;https + + // HTTP scheme (http,https) (default: http) + Scheme string `json:"scheme,omitempty"` + + // +kubebuilder:validation:Enum:=json + + // Serialization (json) (default: json) + Serializer string `json:"serializer,omitempty"` + + // Keys from log event to rewrite for instance from 'kubernetes_namespace' to + // 'k8s_namespace' tags will be rewritten with substring substitution and + // applied in the order present in the hash (Hashes enumerate their values in + // the order that the corresponding keys were inserted see: + // https://ruby-doc.org/core-2.2.2/Hash.html + // + // (default { + // 'kubernetes_':'k8s_', + // 'namespace':'ns', + // 'labels_':'', + // '_name':'', + // '_hash':'', + // 'container_':'' + // }) + ShortenKeys map[string]string `json:"shorten_keys,omitempty"` + + // +docLink:"Secret,../secret/" + Username *secret.Secret `json:"username,omitempty"` +} + +func (in *VMwareLogInsightOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { + vmwLi := types.OutputPlugin{ + PluginMeta: types.PluginMeta{ + Type: "vmware_loginsight", + Id: id, + Directive: "match", + Tag: "**", + }, + } + + if in.Buffer != nil { + bufDirective, err := in.Buffer.ToDirective(secretLoader, id) + if err != nil { + return nil, err + } + + vmwLi.SubDirectives = append(vmwLi.SubDirectives, bufDirective) + } + + params, err := types.NewStructToStringMapper(secretLoader).StringsMap(in) + if err != nil { + return nil, err + } + + vmwLi.Params = params + + return &vmwLi, nil +} diff --git a/pkg/sdk/logging/model/output/vmware_loginsight_test.go b/pkg/sdk/logging/model/output/vmware_loginsight_test.go new file mode 100644 index 0000000000..a065c94872 --- /dev/null +++ b/pkg/sdk/logging/model/output/vmware_loginsight_test.go @@ -0,0 +1,101 @@ +// Copyright © 2019 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package output_test + +import ( + "testing" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/require" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/render" +) + +func TestVMwareLogInsight(t *testing.T) { + // language=yaml + CONFIG := []byte(`--- +agent_id: test_agent_id +authentication: basic +buffer: + disabled: false +config_param: + source: log_source +flatten_hashes: true +flatten_hashes_separator: _ +http_conn_debug: true +http_method: post +host: 127.0.0.1 +log_text_keys: +- log +- message +- msg +max_batch_size: 400000 +path: api/v1/events/ingest +port: 443 +raise_on_error: true +rate_limit_msec: 10 +request_retries: 10 +request_timeout: 10 +ssl_verify: false +scheme: https +serializer: json +shorten_keys: + kubernetes_: k8s_ + namespace: ns + labels_: "" + _name: "" + _hash: "" + container_: ""`) + + expected := ` + + @type vmware_loginsight + @id test + agent_id test_agent_id + authentication basic + config_param {"source":"log_source"} + flatten_hashes true + flatten_hashes_separator _ + host 127.0.0.1 + http_conn_debug true + http_method post + log_text_keys ["log","message","msg"] + max_batch_size 400000 + path api/v1/events/ingest + port 443 + raise_on_error true + rate_limit_msec 10 + request_retries 10 + request_timeout 10 + scheme https + serializer json + shorten_keys {"_hash":"","_name":"","container_":"","kubernetes_":"k8s_","labels_":"","namespace":"ns"} + ssl_verify false + + @type file + chunk_limit_size 8MB + path /buffers/test.*.buffer + retry_forever true + timekey 10m + timekey_wait 1m + + +` + es := &output.VMwareLogInsightOutput{} + require.NoError(t, yaml.Unmarshal(CONFIG, es)) + test := render.NewOutputPluginTest(t, es) + test.DiffResult(expected) +} diff --git a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go index 5d9de5b5ec..a492108ae6 100644 --- a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go @@ -1719,3 +1719,72 @@ func (in *SyslogOutputConfig) DeepCopy() *SyslogOutputConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMwareLogInsightOutput) DeepCopyInto(out *VMwareLogInsightOutput) { + *out = *in + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(string) + **out = **in + } + if in.Buffer != nil { + in, out := &in.Buffer, &out.Buffer + *out = new(Buffer) + (*in).DeepCopyInto(*out) + } + if in.CAFile != nil { + in, out := &in.CAFile, &out.CAFile + *out = new(secret.Secret) + (*in).DeepCopyInto(*out) + } + if in.ConfigParam != nil { + in, out := &in.ConfigParam, &out.ConfigParam + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.FlattenHashes != nil { + in, out := &in.FlattenHashes, &out.FlattenHashes + *out = new(bool) + **out = **in + } + if in.LogTextKeys != nil { + in, out := &in.LogTextKeys, &out.LogTextKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Password != nil { + in, out := &in.Password, &out.Password + *out = new(secret.Secret) + (*in).DeepCopyInto(*out) + } + if in.SSLVerify != nil { + in, out := &in.SSLVerify, &out.SSLVerify + *out = new(bool) + **out = **in + } + if in.ShortenKeys != nil { + in, out := &in.ShortenKeys, &out.ShortenKeys + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Username != nil { + in, out := &in.Username, &out.Username + *out = new(secret.Secret) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMwareLogInsightOutput. +func (in *VMwareLogInsightOutput) DeepCopy() *VMwareLogInsightOutput { + if in == nil { + return nil + } + out := new(VMwareLogInsightOutput) + in.DeepCopyInto(out) + return out +}