diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml index 16f31f2d5..5991bb21e 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml @@ -6557,6 +6557,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: @@ -13115,6 +13258,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml index dccbcf59c..bbbf1c48d 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml @@ -12793,6 +12793,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml index 16f31f2d5..5991bb21e 100644 --- a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml @@ -6557,6 +6557,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: @@ -13115,6 +13258,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml index dccbcf59c..bbbf1c48d 100644 --- a/config/crd/bases/logging.banzaicloud.io_outputs.yaml +++ b/config/crd/bases/logging.banzaicloud.io_outputs.yaml @@ -12793,6 +12793,149 @@ spec: required: - host type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object type: object status: properties: diff --git a/docs/configuration/crds/v1beta1/output_types.md b/docs/configuration/crds/v1beta1/output_types.md index 40b55136b..e50e4cb3f 100644 --- a/docs/configuration/crds/v1beta1/output_types.md +++ b/docs/configuration/crds/v1beta1/output_types.md @@ -95,6 +95,9 @@ OutputSpec defines the desired state of Output ### syslog (*output.SyslogOutputConfig, optional) {#outputspec-syslog} +### vmwareLogIntelligence (*output.VMWareLogIntelligenceOutput, optional) {#outputspec-vmwarelogintelligence} + + ## OutputStatus diff --git a/docs/configuration/plugins/_index.md b/docs/configuration/plugins/_index.md index 1b4f1c919..44ca3c6de 100644 --- a/docs/configuration/plugins/_index.md +++ b/docs/configuration/plugins/_index.md @@ -57,6 +57,7 @@ For more information please click on the plugin name | **[SQS](outputs/sqs/)** | outputs | Output plugin writes fluent-events as queue messages to Amazon SQS | Testing | [v2.1.0](https://github.com/ixixi/fluent-plugin-sqs) | | **[SumoLogic](outputs/sumologic/)** | outputs | Send your logs to Sumologic | GA | [1.8.0](https://github.com/SumoLogic/fluentd-output-sumologic/releases/tag/1.8.0) | | **[Syslog](outputs/syslog/)** | outputs | Output plugin writes events to syslog | GA | [0.9.0.rc.8](https://github.com/cloudfoundry/fluent-plugin-syslog_rfc5424) | +| **[VMwareLogIntelligence](outputs/vmware_log_intelligence/)** | outputs | Send your logs to VMware Log Intelligence | GA | [v2.0.8](https://github.com/vmware/fluent-plugin-vmware-log-intelligence/releases/tag/v2.0.8) | | **[Syslog-NG Match](syslogng-filters/match/)** | syslogng-filters | Selectively keep records | GA | [more info](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/65#TOPIC-1829159) | | **[Syslog-NG Parser](syslogng-filters/parser/)** | syslogng-filters | Parse data from records | GA | [more info](https://axoflow.com/docs/axosyslog-core/chapter-parsers/) | | **[Syslog-NG Rewrite](syslogng-filters/rewrite/)** | syslogng-filters | Rewrite parts of the message | GA | [more info](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/77) | diff --git a/docs/configuration/plugins/outputs/vmware_log_intelligence.md b/docs/configuration/plugins/outputs/vmware_log_intelligence.md new file mode 100644 index 000000000..1d443e9f0 --- /dev/null +++ b/docs/configuration/plugins/outputs/vmware_log_intelligence.md @@ -0,0 +1,111 @@ +--- +title: VMware Log Intelligence +weight: 200 +generated_file: true +--- + +# VMware Log Intelligence output plugin for Fluentd +## Overview + +For details, see [https://github.com/vmware/fluent-plugin-vmware-log-intelligence](https://github.com/vmware/fluent-plugin-vmware-log-intelligence). +## Example output configurations +```yaml +spec: + vmwarelogintelligence: + endpoint_url: https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream + verify_ssl: true + http_compress: false + headers: + Content-Type: "application/json" + Authorization: "Bearer 12345" + structure: simple + buffer: + chunk_limit_records: 300 + flush_interval: 3s + retry_max_times: 3 +``` + + +## Configuration +## VMwareLogIntelligence + +### buffer (*Buffer, optional) {#vmwarelogintelligence-buffer} + +[Buffer](../buffer/) + + +### endpoint_url (string, required) {#vmwarelogintelligence-endpoint_url} + +Log Intelligence endpoint to send logs to https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-endpoint_url + + +### format (*Format, optional) {#vmwarelogintelligence-format} + +[Format](../format/) + + +### http_compress (*bool, optional) {#vmwarelogintelligence-http_compress} + +Compress http request https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-http_compress + + +### headers (LogIntelligenceHeaders, required) {#vmwarelogintelligence-headers} + +Required headers for sending logs to VMware Log Intelligence https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-3Cheaders-3E + + +### verify_ssl (bool, required) {#vmwarelogintelligence-verify_ssl} + +Verify SSL (default: true) https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-verify_ssl + +Default: true + + +## VMwareLogIntelligenceHeaders + +headers +https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-3Cheaders-3E + +### authorization (*secret.Secret, required) {#vmwarelogintelligenceheaders-authorization} + +Authorization Bearer token for http request to VMware Log Intelligence [Secret](../secret/) + + +### content_type (string, required) {#vmwarelogintelligenceheaders-content_type} + +Content Type for http request to VMware Log Intelligence + +Default: application/json + +### structure (string, required) {#vmwarelogintelligenceheaders-structure} + +Structure for http request to VMware Log Intelligence + +Default: simple + + +## LogIntelligenceHeadersOut + +LogIntelligenceHeadersOut is used to convert the input LogIntelligenceHeaders to a fluentd +output that uses the correct key names for the VMware Log Intelligence plugin. This allows the +Ouput to accept the config is snake_case (as other output plugins do) but output the fluentd + config with the proper key names (ie. content_type -> Content-Type) + +### Authorization (*secret.Secret, required) {#logintelligenceheadersout-authorization} + +Authorization Bearer token for http request to VMware Log Intelligence + + +### Content-Type (string, required) {#logintelligenceheadersout-content-type} + +Content Type for http request to VMware Log Intelligence + +Default: application/json + +### structure (string, required) {#logintelligenceheadersout-structure} + +Structure for http request to VMware Log Intelligence + +Default: simple + + diff --git a/pkg/sdk/logging/api/v1beta1/output_types.go b/pkg/sdk/logging/api/v1beta1/output_types.go index f446b7c34..746e812cd 100644 --- a/pkg/sdk/logging/api/v1beta1/output_types.go +++ b/pkg/sdk/logging/api/v1beta1/output_types.go @@ -59,6 +59,7 @@ type OutputSpec struct { SQSOutputConfig *output.SQSOutputConfig `json:"sqs,omitempty"` MattermostOutputConfig *output.MattermostOutputConfig `json:"mattermost,omitempty"` RelabelOutputConfig *output.RelabelOutputConfig `json:"relabel,omitempty"` + VMWareLogIntelligenceOutput *output.VMWareLogIntelligenceOutput `json:"vmwareLogIntelligence,omitempty"` } // OutputStatus defines the observed state of Output diff --git a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go index 2e05c58f2..f7c84e040 100644 --- a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go @@ -2526,6 +2526,11 @@ func (in *OutputSpec) DeepCopyInto(out *OutputSpec) { *out = new(output.RelabelOutputConfig) **out = **in } + if in.VMWareLogIntelligenceOutput != nil { + in, out := &in.VMWareLogIntelligenceOutput, &out.VMWareLogIntelligenceOutput + *out = new(output.VMWareLogIntelligenceOutput) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OutputSpec. diff --git a/pkg/sdk/logging/model/output/vmware_log_intelligence.go b/pkg/sdk/logging/model/output/vmware_log_intelligence.go new file mode 100644 index 000000000..5c7f9b386 --- /dev/null +++ b/pkg/sdk/logging/model/output/vmware_log_intelligence.go @@ -0,0 +1,151 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package output + +import ( + "github.com/cisco-open/operator-tools/pkg/secret" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types" +) + +// +name:"VMware Log Intelligence" +// +weight:"200" +type _hugoVMWareLogIntelligence interface{} //nolint:deadcode,unused + +// +docName:"VMware Log Intelligence output plugin for Fluentd" +/* +For details, see [https://github.com/vmware/fluent-plugin-vmware-log-intelligence](https://github.com/vmware/fluent-plugin-vmware-log-intelligence). +## Example output configurations +```yaml +spec: + vmwarelogintelligence: + endpoint_url: https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream + verify_ssl: true + http_compress: false + headers: + Content-Type: "application/json" + Authorization: "Bearer 12345" + structure: simple + buffer: + chunk_limit_records: 300 + flush_interval: 3s + retry_max_times: 3 +``` +*/ +type _docVMWareLogIntelligence interface{} //nolint:deadcode,unused + +// +name:"VMwareLogIntelligence" +// +url:"https://github.com/vmware/fluent-plugin-vmware-log-intelligence/releases/tag/v2.0.8" +// +version:"v2.0.8" +// +description:"Send your logs to VMware Log Intelligence" +// +status:"GA" +type _metaVMWareLogIntelligence interface{} //nolint:deadcode,unused + +// +kubebuilder:object:generate=true +// +docName:"VMwareLogIntelligence" +type VMWareLogIntelligenceOutput struct { + // Log Intelligence endpoint to send logs to https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-endpoint_url + EndpointURL string `json:"endpoint_url"` + // Verify SSL (default: true) https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-verify_ssl + VerifySSL bool `json:"verify_ssl" plugin:"default:true"` + // Compress http request https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-http_compress + HTTPCompress *bool `json:"http_compress,omitempty"` + // Required headers for sending logs to VMware Log Intelligence https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-3Cheaders-3E + Headers LogIntelligenceHeaders `json:"headers"` + // +docLink:"Buffer,../buffer/" + Buffer *Buffer `json:"buffer,omitempty"` + // +docLink:"Format,../format/" + Format *Format `json:"format,omitempty"` +} + +// +kubebuilder:object:generate=true +// +docName:"VMwareLogIntelligenceHeaders" +// headers +// https://github.com/vmware/fluent-plugin-vmware-log-intelligence?tab=readme-ov-file#label-3Cheaders-3E +type LogIntelligenceHeaders struct { + // Authorization Bearer token for http request to VMware Log Intelligence + // +docLink:"Secret,../secret/" + Authorization *secret.Secret `json:"authorization"` + // Content Type for http request to VMware Log Intelligence + ContentType string `json:"content_type" plugin:"default:application/json"` + // Structure for http request to VMware Log Intelligence + Structure string `json:"structure" plugin:"default:simple"` +} + +// LogIntelligenceHeadersOut is used to convert the input LogIntelligenceHeaders to a fluentd +// output that uses the correct key names for the VMware Log Intelligence plugin. This allows the +// Ouput to accept the config is snake_case (as other output plugins do) but output the fluentd +// config with the proper key names (ie. content_type -> Content-Type) +type LogIntelligenceHeadersOut struct { + // Authorization Bearer token for http request to VMware Log Intelligence + Authorization *secret.Secret `json:"Authorization"` + // Content Type for http request to VMware Log Intelligence + ContentType string `json:"Content-Type" plugin:"default:application/json"` + // Structure for http request to VMware Log Intelligence + Structure string `json:"structure" plugin:"default:simple"` +} + +func (l *LogIntelligenceHeadersOut) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { + return types.NewFlatDirective(types.PluginMeta{ + Directive: "headers", + }, l, secretLoader) +} + +func (v *VMWareLogIntelligenceOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { + const pluginType = "vmware_log_intelligence" + vmwli := &types.OutputPlugin{ + PluginMeta: types.PluginMeta{ + Type: pluginType, + Directive: "match", + Tag: "**", + Id: id, + }, + } + + if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(v); err != nil { + return nil, err + } else { + vmwli.Params = params + } + + h := LogIntelligenceHeadersOut{ + ContentType: v.Headers.ContentType, + Authorization: v.Headers.Authorization, + Structure: v.Headers.Structure, + } + + if headers, err := h.ToDirective(secretLoader, id); err != nil { + return nil, err + } else { + vmwli.SubDirectives = append(vmwli.SubDirectives, headers) + } + + if v.Buffer == nil { + v.Buffer = &Buffer{} + } + if buffer, err := v.Buffer.ToDirective(secretLoader, id); err != nil { + return nil, err + } else { + vmwli.SubDirectives = append(vmwli.SubDirectives, buffer) + } + + if v.Format != nil { + if format, err := v.Format.ToDirective(secretLoader, ""); err != nil { + return nil, err + } else { + vmwli.SubDirectives = append(vmwli.SubDirectives, format) + } + } + return vmwli, nil +} diff --git a/pkg/sdk/logging/model/output/vmware_log_intelligence_test.go b/pkg/sdk/logging/model/output/vmware_log_intelligence_test.go new file mode 100644 index 000000000..fed31efc8 --- /dev/null +++ b/pkg/sdk/logging/model/output/vmware_log_intelligence_test.go @@ -0,0 +1,169 @@ +// Copyright © 2024 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package output_test + +import ( + "testing" + + "github.com/ghodss/yaml" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/render" + + "github.com/stretchr/testify/require" +) + +func TestVMWareLogIntelligenceOutputConfig(t *testing.T) { + CONFIG := []byte(` +endpoint_url: https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream +verify_ssl: true +http_compress: false +headers: + content_type: "application/json" + authorization: + value: "Bearer 12345" + structure: simple +buffer: + chunk_limit_records: 300 + flush_interval: 3s + retry_max_times: 3 +`) + + expected := ` + + @type vmware_log_intelligence + @id test + endpoint_url https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream + http_compress false + verify_ssl true + + Authorization Bearer 12345 + Content-Type application/json + structure simple + + + @type file + chunk_limit_records 300 + chunk_limit_size 8MB + flush_interval 3s + path /buffers/test.*.buffer + retry_forever true + retry_max_times 3 + timekey 10m + timekey_wait 1m + + +` + s := &output.VMWareLogIntelligenceOutput{} + require.NoError(t, yaml.Unmarshal(CONFIG, s)) + test := render.NewOutputPluginTest(t, s) + test.DiffResult(expected) +} + +func TestVMWareLogIntelligenceOutputConfigWithDefaultHeaderValues(t *testing.T) { + CONFIG := []byte(` +endpoint_url: https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream +verify_ssl: true +http_compress: false +headers: + authorization: + value: "Bearer 12345" +buffer: + chunk_limit_records: 300 + flush_interval: 3s + retry_max_times: 3 +`) + + //TODO: test format block + expected := ` + + @type vmware_log_intelligence + @id test + endpoint_url https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream + http_compress false + verify_ssl true + + Authorization Bearer 12345 + Content-Type application/json + structure simple + + + @type file + chunk_limit_records 300 + chunk_limit_size 8MB + flush_interval 3s + path /buffers/test.*.buffer + retry_forever true + retry_max_times 3 + timekey 10m + timekey_wait 1m + + +` + s := &output.VMWareLogIntelligenceOutput{} + require.NoError(t, yaml.Unmarshal(CONFIG, s)) + test := render.NewOutputPluginTest(t, s) + test.DiffResult(expected) +} + +func TestVMWareLogIntelligenceOutputConfigWithFormat(t *testing.T) { + CONFIG := []byte(` +endpoint_url: https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream +verify_ssl: true +headers: + content_type: "application/json" + authorization: + value: "Bearer 12345" + structure: simple +buffer: + chunk_limit_records: 300 + flush_interval: 3s + retry_max_times: 3 +format: + type: json +`) + + //TODO: test format block + expected := ` + + @type vmware_log_intelligence + @id test + endpoint_url https://data.upgrade.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream + verify_ssl true + + Authorization Bearer 12345 + Content-Type application/json + structure simple + + + @type file + chunk_limit_records 300 + chunk_limit_size 8MB + flush_interval 3s + path /buffers/test.*.buffer + retry_forever true + retry_max_times 3 + timekey 10m + timekey_wait 1m + + + @type json + + +` + s := &output.VMWareLogIntelligenceOutput{} + require.NoError(t, yaml.Unmarshal(CONFIG, s)) + test := render.NewOutputPluginTest(t, s) + test.DiffResult(expected) +} diff --git a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go index 5d9de5b5e..97a7398fc 100644 --- a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go @@ -954,6 +954,26 @@ func (in *LogDNAOutput) DeepCopy() *LogDNAOutput { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogIntelligenceHeaders) DeepCopyInto(out *LogIntelligenceHeaders) { + *out = *in + if in.Authorization != nil { + in, out := &in.Authorization, &out.Authorization + *out = new(secret.Secret) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogIntelligenceHeaders. +func (in *LogIntelligenceHeaders) DeepCopy() *LogIntelligenceHeaders { + if in == nil { + return nil + } + out := new(LogIntelligenceHeaders) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogZOutput) DeepCopyInto(out *LogZOutput) { *out = *in @@ -1719,3 +1739,34 @@ func (in *SyslogOutputConfig) DeepCopy() *SyslogOutputConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMWareLogIntelligenceOutput) DeepCopyInto(out *VMWareLogIntelligenceOutput) { + *out = *in + if in.HTTPCompress != nil { + in, out := &in.HTTPCompress, &out.HTTPCompress + *out = new(bool) + **out = **in + } + in.Headers.DeepCopyInto(&out.Headers) + if in.Buffer != nil { + in, out := &in.Buffer, &out.Buffer + *out = new(Buffer) + (*in).DeepCopyInto(*out) + } + if in.Format != nil { + in, out := &in.Format, &out.Format + *out = new(Format) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMWareLogIntelligenceOutput. +func (in *VMWareLogIntelligenceOutput) DeepCopy() *VMWareLogIntelligenceOutput { + if in == nil { + return nil + } + out := new(VMWareLogIntelligenceOutput) + in.DeepCopyInto(out) + return out +}