diff --git a/docs/configuration/plugins/_index.md b/docs/configuration/plugins/_index.md index 51e5f348e..e95a7497c 100644 --- a/docs/configuration/plugins/_index.md +++ b/docs/configuration/plugins/_index.md @@ -13,7 +13,7 @@ For more information please click on the plugin name | **[Concat](filters/concat/)** | filters | Fluentd Filter plugin to concatenate multiline log separated in multiple events. | GA | [2.5.0](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) | | **[Dedot](filters/dedot/)** | filters | Concatenate multiline log separated in multiple events | GA | [1.0.0](https://github.com/lunardial/fluent-plugin-dedot_filter) | | **[Exception Detector](filters/detect_exceptions/)** | filters | Exception Detector | GA | [0.0.14](https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions) | -| **[ElasticsearchGenId](filters/elasticsearch_genid/)** | filters | | | []() | +| **[ElasticSearch GenId](filters/elasticsearch_genid/)** | filters | | | [](TODO) | | **[Enhance K8s Metadata](filters/enhance_k8s/)** | filters | Fluentd output plugin to add extra Kubernetes metadata to the events. | GA | [2.0.0](https://github.com/SumoLogic/sumologic-kubernetes-collection/tree/main/fluent-plugin-enhance-k8s-metadata) | | **[Geo IP](filters/geoip/)** | filters | Fluentd GeoIP filter | GA | [1.3.2](https://github.com/y-ken/fluent-plugin-geoip) | | **[Grep](filters/grep/)** | filters | Grep events by the values | GA | [more info](https://docs.fluentd.org/filter/grep) | diff --git a/docs/configuration/plugins/filters/concat.md b/docs/configuration/plugins/filters/concat.md index ad0dcb759..995a32162 100644 --- a/docs/configuration/plugins/filters/concat.md +++ b/docs/configuration/plugins/filters/concat.md @@ -55,7 +55,7 @@ Default: - ### flush_interval (int, optional) {#concat-flush_interval} -The number of seconds after which the last received event log will be flushed. If specified 0, wait for next line forever. +The number of seconds after which the last received event log is flushed. If set to 0, flushing is disabled (wait for next line forever). Default: - @@ -126,38 +126,37 @@ The key name that is referred to detect stream name on cri log Default: - - ## Example `Concat` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Concat` filter configurations - spec: +// {{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - concat: + partial_key: "partial_message" + separator: "" + n_lines: 10 + selectors: {} + localOutputRefs: + - demo-output +// {{< /highlight >}} - filters: - - concat: - partial_key: "partial_message" - separator: "" - n_lines: 10 - selectors: {} - localOutputRefs: - - demo-output - ``` +#### Fluentd config result: - #### Fluentd Config Result - ```yaml - +{{< highlight xml >}} + + @type concat + @id test_concat + key message + n_lines 10 + partial_key partial_message + +{{< /highlight >}} - @type concat - @id test_concat - key message - n_lines 10 - partial_key partial_message - - - ``` --- diff --git a/docs/configuration/plugins/filters/dedot.md b/docs/configuration/plugins/filters/dedot.md index a374babfd..5a4e16bb7 100644 --- a/docs/configuration/plugins/filters/dedot.md +++ b/docs/configuration/plugins/filters/dedot.md @@ -13,7 +13,7 @@ generated_file: true ### de_dot_nested (bool, optional) {#dedotfilterconfig-de_dot_nested} -Will cause the plugin to recurse through nested structures (hashes and arrays), and remove dots in those key-names too. +Will cause the plugin to recourse through nested structures (hashes and arrays), and remove dots in those key-names too. Default: false @@ -24,36 +24,34 @@ Separator Default: _ - ## Example `Dedot` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Dedot` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - dedot: + de_dot_separator: "-" + de_dot_nested: true + selectors: {} + localOutputRefs: + - demo-output +{{}} - spec: - filters: - - dedot: - de_dot_separator: "-" - de_dot_nested: true - selectors: {} - localOutputRefs: - - demo-output +#### Fluentd config result: - ``` +{{< highlight xml >}} + + @type dedot + @id test_dedot + de_dot_nested true + de_dot_separator - + +{{}} - #### Fluentd Config Result - ```yaml - - - @type dedot - @id test_dedot - de_dot_nested true - de_dot_separator - - - - ``` --- diff --git a/docs/configuration/plugins/filters/detect_exceptions.md b/docs/configuration/plugins/filters/detect_exceptions.md index 5ddddc16a..af5af79bd 100644 --- a/docs/configuration/plugins/filters/detect_exceptions.md +++ b/docs/configuration/plugins/filters/detect_exceptions.md @@ -45,7 +45,7 @@ Default: nil Programming languages for which to detect exceptions. -Default: [] +Default: `[]` ### max_lines (int, optional) {#detectexceptions-max_lines} @@ -75,42 +75,40 @@ Default: false Tag used in match directive. -Default: kubernetes.** +Default: `kubernetes.**` + + + +## Example `Exception Detector` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - detectExceptions: + multiline_flush_interval: 0.1 + languages: + - java + - python + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml >}} + + @type detect_exceptions + @id test_detect_exceptions + languages ["java","python"] + multiline_flush_interval 0.1 + remove_tag_prefix kubernetes + +{{}} - ## Example `Exception Detector` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - - spec: - - filters: - - detectExceptions: - multiline_flush_interval: 0.1 - languages: - - java - - python - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type detect_exceptions - @id test_detect_exceptions - languages ["java","python"] - multiline_flush_interval 0.1 - remove_tag_prefix kubernetes - - - ``` - --- diff --git a/docs/configuration/plugins/filters/elasticsearch_genid.md b/docs/configuration/plugins/filters/elasticsearch_genid.md index 86e76636f..1f71b406e 100644 --- a/docs/configuration/plugins/filters/elasticsearch_genid.md +++ b/docs/configuration/plugins/filters/elasticsearch_genid.md @@ -1,3 +1,37 @@ +--- +title: ElasticSearch GenId +weight: 200 +generated_file: true +--- + +# ElasticSearch GenId +## Overview + +TODO: FILL +// ## Example `Elasticsearch Genid` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - elasticsearch_genid: + hash_id_key: gen_id + selectors: {} + localOutputRefs: + - demo-output + +#### Fluentd Config Result + + @type elasticsearch_genid + @id test_elasticsearch_genid + hash_id_key gen_id + +{{}} + + +## Configuration ## ElasticsearchGenId ### hash_id_key (string, optional) {#elasticsearchgenid-hash_id_key} diff --git a/docs/configuration/plugins/filters/enhance_k8s.md b/docs/configuration/plugins/filters/enhance_k8s.md index 669429af2..817d49354 100644 --- a/docs/configuration/plugins/filters/enhance_k8s.md +++ b/docs/configuration/plugins/filters/enhance_k8s.md @@ -15,11 +15,11 @@ generated_file: true parameters for read/write record -Default: ['$.namespace'] +Default: `['$.namespace']` ### in_pod_path ([]string, optional) {#enhancek8s-in_pod_path} -Default: ['$.pod','$.pod_name'] +Default: `['$.pod','$.pod_name']` ### data_type (string, optional) {#enhancek8s-data_type} @@ -79,11 +79,11 @@ Default: ['v1'] Kubernetes resources api groups -Default: ["apps/v1", "extensions/v1beta1"] +Default: `["apps/v1", "extensions/v1beta1"]` ### ssl_partial_chain (*bool, optional) {#enhancek8s-ssl_partial_chain} -if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want to trust the intermediate CA certs we do have, set this to `true` - this corresponds to the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN +If `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want to trust the intermediate CA certs we do have, set this to `true` - this corresponds to the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN Default: false @@ -112,29 +112,28 @@ Cache refresh variation Default: 60*15 - ## Example `EnhanceK8s` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Logging - metadata: - name: demo-flow +## Example `EnhanceK8s` filter configurations - spec: +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: demo-flow +spec: + globalFilters: + - enhanceK8s: {} +{{}} - globalFilters: - - enhanceK8s: {} - ``` +#### Fluentd config result: - #### Fluentd Config Result - ```yaml - +{{< highlight xml >}} + + @type enhance_k8s_metadata + @id test_enhanceK8s + +{{}} - @type enhance_k8s_metadata - @id test_enhanceK8s - - - ``` --- diff --git a/docs/configuration/plugins/filters/geoip.md b/docs/configuration/plugins/filters/geoip.md index 3a906c67d..f023b44b0 100644 --- a/docs/configuration/plugins/filters/geoip.md +++ b/docs/configuration/plugins/filters/geoip.md @@ -49,48 +49,47 @@ Records are represented as maps: `key: value` Default: - - ## Example `GeoIP` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - - spec: - - filters: - - geoip: - geoip_lookup_keys: remote_addr - records: - - city: ${city.names.en["remote_addr"]} - location_array: '''[${location.longitude["remote"]},${location.latitude["remote"]}]''' - country: ${country.iso_code["remote_addr"]} - country_name: ${country.names.en["remote_addr"]} - postal_code: ${postal.code["remote_addr"]} - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type geoip - @id test_geoip - geoip_lookup_keys remote_addr - skip_adding_null_record true - - city ${city.names.en["remote_addr"]} - country ${country.iso_code["remote_addr"]} - country_name ${country.names.en["remote_addr"]} - location_array '[${location.longitude["remote"]},${location.latitude["remote"]}]' - postal_code ${postal.code["remote_addr"]} - - - - ``` + +## Example `GeoIP` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - geoip: + geoip_lookup_keys: remote_addr + records: + - city: ${city.names.en["remote_addr"]} + location_array: '''[${location.longitude["remote"]},${location.latitude["remote"]}]''' + country: ${country.iso_code["remote_addr"]} + country_name: ${country.names.en["remote_addr"]} + postal_code: ${postal.code["remote_addr"]} + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml >}} + + @type geoip + @id test_geoip + geoip_lookup_keys remote_addr + skip_adding_null_record true + + city ${city.names.en["remote_addr"]} + country ${country.iso_code["remote_addr"]} + country_name ${country.names.en["remote_addr"]} + location_array '[${location.longitude["remote"]},${location.latitude["remote"]}]' + postal_code ${postal.code["remote_addr"]} + + +{{}} + --- diff --git a/docs/configuration/plugins/filters/grep.md b/docs/configuration/plugins/filters/grep.md index 6fd280e55..4d5620ce8 100644 --- a/docs/configuration/plugins/filters/grep.md +++ b/docs/configuration/plugins/filters/grep.md @@ -53,40 +53,39 @@ Pattern expression to evaluate Default: - - ## Example `Regexp` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Regexp` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + regexp: + - key: first + pattern: /^5\d\d$/ + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml >}} + + @type grep + @id demo-flow_1_grep + + key first + pattern /^5\d\d$/ + + +{{}} - spec: - - filters: - - grep: - regexp: - - key: first - pattern: /^5\d\d$/ - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type grep - @id demo-flow_1_grep - - key first - pattern /^5\d\d$/ - - - - ``` --- ## Exclude Directive @@ -106,40 +105,40 @@ Pattern expression to evaluate Default: - - ## Example `Exclude` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Exclude` filter configurations - spec: +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + exclude: + - key: first + pattern: /^5\d\d$/ + selectors: {} + localOutputRefs: + - demo-output +{{}} - filters: - - grep: - exclude: - - key: first - pattern: /^5\d\d$/ - selectors: {} - localOutputRefs: - - demo-output - ``` - #### Fluentd Config Result - ```yaml +#### Fluentd config result: - - @type grep - @id demo-flow_0_grep - - key first - pattern /^5\d\d$/ - - +{{< highlight xml >}} + + @type grep + @id demo-flow_0_grep + + key first + pattern /^5\d\d$/ + + +{{}} - ``` --- ## Or Directive @@ -159,46 +158,45 @@ Default: - Default: - - ## Example `Or` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Or` filter configurations - spec: +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + or: + - exclude: + - key: first + pattern: /^5\d\d$/ + - key: second + pattern: /\.css$/ - filters: - - grep: - or: - - exclude: - - key: first - pattern: /^5\d\d$/ - - key: second - pattern: /\.css$/ + selectors: {} + localOutputRefs: + - demo-output +{{}} - selectors: {} - localOutputRefs: - - demo-output - ``` +#### Fluentd config result: - #### Fluentd Config Result - ```yaml - - - - key first - pattern /^5\d\d$/ - - - key second - pattern /\.css$/ - - +{{< highlight xml >}} + + + key first + pattern /^5\d\d$/ + + + key second + pattern /\.css$/ + + +{{}} - ``` --- ## And Directive @@ -218,34 +216,33 @@ Default: - Default: - - ## Example `And` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - spec: +## Example `And` filter configurations - filters: - - grep: - and: - - regexp: - - key: first - pattern: /^5\d\d$/ - - key: second - pattern: /\.css$/ +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + and: + - regexp: + - key: first + pattern: /^5\d\d$/ + - key: second + pattern: /\.css$/ - selectors: {} - localOutputRefs: - - demo-output + selectors: {} + localOutputRefs: + - demo-output +{{}} - ``` - #### Fluentd Config Result - ```yaml +Fluentd #### config result: +{{< highlight xml >}} key first @@ -256,7 +253,7 @@ Default: - pattern /\.css$/ +{{}} - ``` --- diff --git a/docs/configuration/plugins/filters/kube_events_timestamp.md b/docs/configuration/plugins/filters/kube_events_timestamp.md index 50c6a9293..c97b382a3 100644 --- a/docs/configuration/plugins/filters/kube_events_timestamp.md +++ b/docs/configuration/plugins/filters/kube_events_timestamp.md @@ -24,37 +24,38 @@ Added time field name Default: triggerts - ## Example `Kubernetes Events Timestamp` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: es-flow - - spec: - - filters: - - kube_events_timestamp: - timestamp_fields: - - "event.eventTime" - - "event.lastTimestamp" - - "event.firstTimestamp" - mapped_time_key: mytimefield - selectors: {} - localOutputRefs: - - es-output - - ``` - - #### Fluentd Config Result - ```yaml + +## Example `Kubernetes Events Timestamp` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: es-flow +spec: + filters: + - kube_events_timestamp: + timestamp_fields: + - "event.eventTime" + - "event.lastTimestamp" + - "event.firstTimestamp" + mapped_time_key: mytimefield + selectors: {} + localOutputRefs: + - es-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml >}} @type kube_events_timestamp @id test-kube-events-timestamp timestamp_fields ["event.eventTime","event.lastTimestamp","event.firstTimestamp"] mapped_time_key mytimefield - ``` +{{}} + --- diff --git a/docs/configuration/plugins/filters/parser.md b/docs/configuration/plugins/filters/parser.md index 3f4411482..06fbb5d58 100644 --- a/docs/configuration/plugins/filters/parser.md +++ b/docs/configuration/plugins/filters/parser.md @@ -399,57 +399,56 @@ Use specified timezone. one can parse/format the time value in the specified tim Default: - - ## Example `Parser` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - - spec: - - filters: - - parser: - remove_key_name_field: true - reserve_data: true - parse: - type: multi_format - patterns: - - format: nginx - - format: regexp - expression: /foo/ - - format: none - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type parser - @id test_parser - key_name message - remove_key_name_field true - reserve_data true - - @type multi_format - - format nginx - - - expression /foo/ - format regexp - - - format none - - - - - ``` + +## Example `Parser` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - parser: + remove_key_name_field: true + reserve_data: true + parse: + type: multi_format + patterns: + - format: nginx + - format: regexp + expression: /foo/ + - format: none + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight yaml >}} + + @type parser + @id test_parser + key_name message + remove_key_name_field true + reserve_data true + + @type multi_format + + format nginx + + + expression /foo/ + format regexp + + + format none + + + +{{}} + --- diff --git a/docs/configuration/plugins/filters/prometheus.md b/docs/configuration/plugins/filters/prometheus.md index 24881a01d..3b8b442f7 100644 --- a/docs/configuration/plugins/filters/prometheus.md +++ b/docs/configuration/plugins/filters/prometheus.md @@ -61,61 +61,59 @@ Additional labels for this metric Default: - - ## Example `Prometheus` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - - spec: - - filters: - - tag_normaliser: {} - - parser: - remove_key_name_field: true - reserve_data: true - parse: - type: nginx - - prometheus: - metrics: - - name: total_counter - desc: The total number of foo in message. - type: counter - labels: - foo: bar - labels: - host: ${hostname} - tag: ${tag} - namespace: $.kubernetes.namespace - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ``` - - - @type prometheus - @id logging-demo-flow_2_prometheus - - desc The total number of foo in message. - name total_counter - type counter - - foo bar - - - - host ${hostname} - namespace $.kubernetes.namespace - tag ${tag} - - - - ``` + +## Example `Prometheus` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - tag_normaliser: {} + - parser: + remove_key_name_field: true + reserve_data: true + parse: + type: nginx + - prometheus: + metrics: + - name: total_counter + desc: The total number of foo in message. + type: counter + labels: + foo: bar + labels: + host: ${hostname} + tag: ${tag} + namespace: $.kubernetes.namespace + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml>}} + + @type prometheus + @id logging-demo-flow_2_prometheus + + desc The total number of foo in message. + name total_counter + type counter + + foo bar + + + + host ${hostname} + namespace $.kubernetes.namespace + tag ${tag} + + +{{}} + --- diff --git a/docs/configuration/plugins/filters/record_modifier.md b/docs/configuration/plugins/filters/record_modifier.md index 69d7ca31a..214b1944b 100644 --- a/docs/configuration/plugins/filters/record_modifier.md +++ b/docs/configuration/plugins/filters/record_modifier.md @@ -13,7 +13,7 @@ generated_file: true ### prepare_value (string, optional) {#recordmodifier-prepare_value} -Prepare values for filtering in configure phase. Prepared values can be used in . You can write any ruby code. +Prepare values for filtering in configure phase. Prepared values can be used in ``. You can write any ruby code. Default: - @@ -48,38 +48,36 @@ Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier Default: - - ## Example `Record Modifier` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Record Modifier` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - record_modifier: + records: + - foo: "bar" + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight xml >}} + + @type record_modifier + @id test_record_modifier + + foo bar + + +{{}} - spec: - - filters: - - record_modifier: - records: - - foo: "bar" - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type record_modifier - @id test_record_modifier - - foo bar - - - - ``` --- ## [Replace Directive](https://github.com/repeatedly/fluent-plugin-record-modifier#replace_keys_value) diff --git a/docs/configuration/plugins/filters/record_transformer.md b/docs/configuration/plugins/filters/record_transformer.md index ff93b54ca..9c4376290 100644 --- a/docs/configuration/plugins/filters/record_transformer.md +++ b/docs/configuration/plugins/filters/record_transformer.md @@ -37,7 +37,7 @@ Default: - ### enable_ruby (bool, optional) {#recordtransformer-enable_ruby} -When set to true, the full Ruby syntax is enabled in the ${...} expression. +When set to true, the full Ruby syntax is enabled in the `${...}` expression. Default: false @@ -54,37 +54,37 @@ Add records docs at: https://docs.fluentd.org/filter/record_transformer Records Default: - - ## Example `Record Transformer` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Record Transformer` filter configurations - spec: +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - record_transformer: + records: + - foo: "bar" + selectors: {} + localOutputRefs: + - demo-output +{{}} - filters: - - record_transformer: - records: - - foo: "bar" - selectors: {} - localOutputRefs: - - demo-output - ``` - #### Fluentd Config Result - ```yaml - +Fluentd config result: - @type record_transformer - @id test_record_transformer - - foo bar - +{{< highlight xml >}} + + @type record_transformer + @id test_record_transformer + + foo bar + + +{{}} - - ``` --- diff --git a/docs/configuration/plugins/filters/stdout.md b/docs/configuration/plugins/filters/stdout.md index 18edd29d8..c0e8d6a14 100644 --- a/docs/configuration/plugins/filters/stdout.md +++ b/docs/configuration/plugins/filters/stdout.md @@ -18,34 +18,33 @@ This is the option of stdout format. Default: - - ## Example `StdOut` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `StdOut` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - stdout: + output_type: json + selectors: {} + localOutputRefs: + - demo-output +{{}} + + +#### Fluentd config result: + +{{< highlight yaml >}} + + @type stdout + @id test_stdout + output_type json + +{{}} - spec: - - filters: - - stdout: - output_type: json - selectors: {} - localOutputRefs: - - demo-output - - ``` - - #### Fluentd Config Result - ```yaml - - - @type stdout - @id test_stdout - output_type json - - - ``` --- diff --git a/docs/configuration/plugins/filters/sumologic.md b/docs/configuration/plugins/filters/sumologic.md index 6c7b63680..55dea91b7 100644 --- a/docs/configuration/plugins/filters/sumologic.md +++ b/docs/configuration/plugins/filters/sumologic.md @@ -15,7 +15,7 @@ generated_file: true Source Category -Default: "%{namespace}/%{pod_name}" +Default: `%{namespace}/%{pod_name}` ### source_category_replace_dash (string, optional) {#sumologic-source_category_replace_dash} @@ -33,7 +33,7 @@ Default: kubernetes/ Source Name -Default: "%{namespace}.%{pod}.%{container}" +Default: `%{namespace}.%{pod}.%{container}` ### log_format (string, optional) {#sumologic-log_format} @@ -129,37 +129,37 @@ Default: "hostname" Tracing Label Prefix -Default: "pod_label_" +Default: `pod_label_` ### tracing_annotation_prefix (string, optional) {#sumologic-tracing_annotation_prefix} Tracing Annotation Prefix -Default: "pod_annotation_" +Default: `pod_annotation_` ### source_host_key_name (string, optional) {#sumologic-source_host_key_name} Source HostKey Name -Default: "_sourceHost" +Default: `_sourceHost` ### source_category_key_name (string, optional) {#sumologic-source_category_key_name} Source CategoryKey Name -Default: "_sourceCategory" +Default: `_sourceCategory` ### source_name_key_name (string, optional) {#sumologic-source_name_key_name} Source NameKey Name -Default: "_sourceName" +Default: `_sourceName` ### collector_key_name (string, optional) {#sumologic-collector_key_name} CollectorKey Name -Default: "_collector" +Default: `_collector` ### collector_value (string, optional) {#sumologic-collector_value} @@ -168,34 +168,33 @@ Collector Value Default: "undefined" - ## Example `Parser` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow +## Example `Parser` filter configurations - spec: +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - sumologic: + source_name: "elso" + selectors: {} + localOutputRefs: + - demo-output +{{}} - filters: - - sumologic: - source_name: "elso" - selectors: {} - localOutputRefs: - - demo-output - ``` +#### Fluentd config result: - #### Fluentd Config Result - ```yaml - +{{< highlight xml >}} + + @type kubernetes_sumologic + @id test_sumologic + source_name elso + +{{}} - @type kubernetes_sumologic - @id test_sumologic - source_name elso - - - ``` --- diff --git a/docs/configuration/plugins/filters/tagnormaliser.md b/docs/configuration/plugins/filters/tagnormaliser.md index 827a2111f..828b4be1c 100644 --- a/docs/configuration/plugins/filters/tagnormaliser.md +++ b/docs/configuration/plugins/filters/tagnormaliser.md @@ -6,19 +6,22 @@ generated_file: true # Fluentd Plugin to re-tag based on log metadata ## Overview - More info at https://github.com/kube-logging/fluent-plugin-tag-normaliser - # Available kubernetes metadata +More info at https://github.com/kube-logging/fluent-plugin-tag-normaliser + +## Available Kubernetes metadata + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `${pod_name}` | Pod name | understood-butterfly-logging-demo-7dcdcfdcd7-h7p9n | +| `${container_name}` | Container name inside the Pod | logging-demo | +| `${namespace_name}` | Namespace name | default | +| `${pod_id}` | Kubernetes UUID for Pod | 1f50d309-45a6-11e9-b795-025000000001 | +| `${labels}` | Kubernetes Pod labels. This is a nested map. You can access nested attributes via `.` | `{"app":"logging-demo", "pod-template-hash":"7dcdcfdcd7" }` | +| `${host}` | Node hostname the Pod runs on | docker-desktop | +| `${docker_id}` | Docker UUID of the container | 3a38148aa37aa3... | + - | Parameter | Description | Example | - |-----------|-------------|---------| - | ${pod_name} | Pod name | understood-butterfly-logging-demo-7dcdcfdcd7-h7p9n | - | ${container_name} | Container name inside the Pod | logging-demo | - | ${namespace_name} | Namespace name | default | - | ${pod_id} | Kubernetes UUID for Pod | 1f50d309-45a6-11e9-b795-025000000001 | - | ${labels} | Kubernetes Pod labels. This is a nested map. You can access nested attributes via `.` | {"app":"logging-demo", "pod-template-hash":"7dcdcfdcd7" } | - | ${host} | Node hostname the Pod runs on | docker-desktop | - | ${docker_id} | Docker UUID of the container | 3a38148aa37aa3... | ## Configuration ## Tag Normaliser parameters @@ -33,37 +36,36 @@ Default: ${namespace_name}.${pod_name}.${container_name} Tag used in match directive. -Default: kubernetes.** - +Default: `kubernetes.**` - ## Example `Parser` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - name: demo-flow - spec: +## Example `Parser` filter configurations - filters: - - tag_normaliser: - format: cluster1.${namespace_name}.${pod_name}.${labels.app} - selectors: {} - localOutputRefs: - - demo-output +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - tag_normaliser: + format: cluster1.${namespace_name}.${pod_name}.${labels.app} + selectors: {} + localOutputRefs: + - demo-output +{{}} - ``` - #### Fluentd Config Result - ```yaml - +#### Fluentd config result: - @type tag_normaliser - @id test_tag_normaliser - format cluster1.${namespace_name}.${pod_name}.${labels.app} +{{< highlight xml >}} + + @type tag_normaliser + @id test_tag_normaliser + format cluster1.${namespace_name}.${pod_name}.${labels.app} + +{{}} - - ``` --- diff --git a/docs/configuration/plugins/filters/throttle.md b/docs/configuration/plugins/filters/throttle.md index a82f99f6d..716f7ba0f 100644 --- a/docs/configuration/plugins/filters/throttle.md +++ b/docs/configuration/plugins/filters/throttle.md @@ -1,3 +1,9 @@ +--- +title: Throttle +weight: 200 +generated_file: true +--- + # [Throttle Filter](https://github.com/rubrikinc/fluent-plugin-throttle) ## Overview A sentry plugin to throttle logs. Logs are grouped by a configurable key. When a group exceeds a configuration rate, logs are dropped for this group. @@ -42,34 +48,33 @@ When a group reaches its limit and as long as it is not reset, a warning message Default: 10 seconds - ## Example `Throttle` filter configurations - ```yaml - apiVersion: logging.banzaicloud.io/v1beta1 - kind: Flow - metadata: - - name: demo-flow - spec: +## Example `Throttle` filter configurations - filters: - - throttle: - group_key: "$.kubernetes.container_name" - selectors: {} - localOutputRefs: - - demo-output +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - throttle: + group_key: "$.kubernetes.container_name" + selectors: {} + localOutputRefs: + - demo-output +{{}} - ``` - #### Fluentd Config Result - ```yaml - +#### Fluentd config result: - @type throttle - @id test_throttle - group_key $.kubernetes.container_name +{{< highlight xml >}} + + @type throttle + @id test_throttle + group_key $.kubernetes.container_name + +{{}} - - ``` --- diff --git a/docs/configuration/plugins/syslogng-filters/match.md b/docs/configuration/plugins/syslogng-filters/match.md index d984f3ec4..6677968cd 100644 --- a/docs/configuration/plugins/syslogng-filters/match.md +++ b/docs/configuration/plugins/syslogng-filters/match.md @@ -6,21 +6,23 @@ generated_file: true # Match ## Overview - Match filters can be used to select the log records to process. These filters have the same options and syntax as the [syslog-ng flow match expressions]({{< relref "/docs/configuration/plugins/syslog-ng-filters/match.md" >}}). - -{{< highlight yaml >}} - filters: - - match: - or: - - regexp: - value: json.kubernetes.labels.app.kubernetes.io/name - pattern: apache - type: string - - regexp: - value: json.kubernetes.labels.app.kubernetes.io/name - pattern: nginx - type: string -{{}} + Match filters can be used to select the log records to process. These filters have the same options and syntax as [syslog-ng flow match expressions]({{< relref "/docs/configuration/plugins/syslogng-filters/match.md" >}}). +// +// {{< highlight yaml >}} +// +// filters: +// - match: +// or: +// - regexp: +// value: json.kubernetes.labels.app.kubernetes.io/name +// pattern: apache +// type: string +// - regexp: +// value: json.kubernetes.labels.app.kubernetes.io/name +// pattern: nginx +// type: string +// +// {{}} ## Configuration ## MatchExpr diff --git a/go.work.sum b/go.work.sum index f9929a006..92a253c69 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1645,6 +1645,7 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1 h1:erE0rdztuaDq3bpGifD95wfoPrSZc95nGA6tbiNYh6M= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -2510,14 +2511,18 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/code-generator v0.28.4/go.mod h1:OQAfl6bZikQ/tK6faJ18Vyzo54rUII2NmjurHyiN1g4= +k8s.io/component-helpers v0.28.2/go.mod h1:pF1R5YWQ+sgf0i6EbVm+MQCzkYuqutDUibdrkvAa6aI= +k8s.io/cri-api v0.27.1/go.mod h1:+Ts/AVYbIo04S86XbTD73UPp/DkTiYxtsFeOFEu32L0= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.28.4/go.mod h1:HL4/lR/bhjAJPbqycKtfhWiKh1Sp21cpHOL8P4oo87w= +k8s.io/metrics v0.28.2/go.mod h1:QTIIdjMrq+KodO+rmp6R9Pr1LZO8kTArNtkWoQXw0sw= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= +sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3/go.mod h1:/d88dHCvoy7d0AKFT0yytezSGZKjsZBVs9YTkBHSGFk= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/sdk/logging/model/filter/concat.go b/pkg/sdk/logging/model/filter/concat.go index 3ff99f1ac..8aeb36658 100644 --- a/pkg/sdk/logging/model/filter/concat.go +++ b/pkg/sdk/logging/model/filter/concat.go @@ -52,7 +52,7 @@ type Concat struct { ContinuousLineRegexp string `json:"continuous_line_regexp,omitempty"` //The key to determine which stream an event belongs to. StreamIdentityKey string `json:"stream_identity_key,omitempty"` - //The number of seconds after which the last received event log will be flushed. If specified 0, wait for next line forever. + //The number of seconds after which the last received event log is flushed. If set to 0, flushing is disabled (wait for next line forever). FlushInterval int `json:"flush_interval,omitempty"` //The label name to handle events caused by timeout. TimeoutLabel string `json:"timeout_label,omitempty"` @@ -78,39 +78,38 @@ type Concat struct { PartialCriStreamKey string `json:"partial_cri_stream_key,omitempty"` } -// ## Example `Concat` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - concat: -// partial_key: "partial_message" -// separator: "" -// n_lines: 10 -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type concat -// @id test_concat -// key message -// n_lines 10 -// partial_key partial_message -// -// -// ``` +/* +## Example `Concat` filter configurations + +// {{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - concat: + partial_key: "partial_message" + separator: "" + n_lines: 10 + selectors: {} + localOutputRefs: + - demo-output +// {{< /highlight >}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type concat + @id test_concat + key message + n_lines 10 + partial_key partial_message + +{{< /highlight >}} +*/ type _expConcat interface{} //nolint:deadcode,unused func (c *Concat) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/dedot.go b/pkg/sdk/logging/model/filter/dedot.go index bacb6e04f..bdfbeddc8 100644 --- a/pkg/sdk/logging/model/filter/dedot.go +++ b/pkg/sdk/logging/model/filter/dedot.go @@ -37,43 +37,41 @@ type _metaDedot interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true type DedotFilterConfig struct { - // Will cause the plugin to recurse through nested structures (hashes and arrays), and remove dots in those key-names too.(default: false) + // Will cause the plugin to recourse through nested structures (hashes and arrays), and remove dots in those key-names too.(default: false) Nested bool `json:"de_dot_nested,omitempty"` // Separator (default:_) Separator string `json:"de_dot_separator,omitempty"` } -// ## Example `Dedot` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - dedot: -// de_dot_separator: "-" -// de_dot_nested: true -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type dedot -// @id test_dedot -// de_dot_nested true -// de_dot_separator - -// -// -// ``` +/* +## Example `Dedot` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - dedot: + de_dot_separator: "-" + de_dot_nested: true + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type dedot + @id test_dedot + de_dot_nested true + de_dot_separator - + +{{}} +*/ type _expDedot interface{} //nolint:deadcode,unused func NewDedotFilterConfig() *DedotFilterConfig { diff --git a/pkg/sdk/logging/model/filter/detect_exceptions.go b/pkg/sdk/logging/model/filter/detect_exceptions.go index 8f46f370b..364c7a8c7 100644 --- a/pkg/sdk/logging/model/filter/detect_exceptions.go +++ b/pkg/sdk/logging/model/filter/detect_exceptions.go @@ -54,7 +54,7 @@ type DetectExceptions struct { RemoveTagPrefix string `json:"remove_tag_prefix,omitempty"` // The interval of flushing the buffer for multiline format. (default: nil) MultilineFlushInterval string `json:"multiline_flush_interval,omitempty"` - // Programming languages for which to detect exceptions. (default: []) + // Programming languages for which to detect exceptions. (default: `[]`) Languages []string `json:"languages,omitempty"` // Maximum number of lines to flush (0 means no limit) (default: 1000) MaxLines int `json:"max_lines,omitempty"` @@ -64,44 +64,42 @@ type DetectExceptions struct { Stream string `json:"stream,omitempty"` // Force line breaks between each lines when comibining exception stacks. (default: false) ForceLineBreaks bool `json:"force_line_breaks,omitempty"` - // Tag used in match directive. (default: kubernetes.**) + // Tag used in match directive. (default: `kubernetes.**`) MatchTag string `json:"match_tag,omitempty" plugin:"hidden"` } -// ## Example `Exception Detector` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - detectExceptions: -// multiline_flush_interval: 0.1 -// languages: -// - java -// - python -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type detect_exceptions -// @id test_detect_exceptions -// languages ["java","python"] -// multiline_flush_interval 0.1 -// remove_tag_prefix kubernetes -// -// -// ``` +/* +## Example `Exception Detector` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - detectExceptions: + multiline_flush_interval: 0.1 + languages: + - java + - python + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type detect_exceptions + @id test_detect_exceptions + languages ["java","python"] + multiline_flush_interval 0.1 + remove_tag_prefix kubernetes + +{{}} +*/ type _expDetectExceptions interface{} //nolint:deadcode,unused func (d *DetectExceptions) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/elasticsearch_genid.go b/pkg/sdk/logging/model/filter/elasticsearch_genid.go index b2ce01bb9..fe87208e0 100644 --- a/pkg/sdk/logging/model/filter/elasticsearch_genid.go +++ b/pkg/sdk/logging/model/filter/elasticsearch_genid.go @@ -19,6 +19,43 @@ import ( "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types" ) +// +name:"ElasticSearch GenId" +// +weight:"200" +type _hugoElasticsearchGenId interface{} //nolint:deadcode,unused + +// +docName:"ElasticSearch GenId" +/* +TODO: FILL +// ## Example `Elasticsearch Genid` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - elasticsearch_genid: + hash_id_key: gen_id + selectors: {} + localOutputRefs: + - demo-output + +#### Fluentd Config Result + + @type elasticsearch_genid + @id test_elasticsearch_genid + hash_id_key gen_id + +{{}} +*/ +type _docElasticsearchGenId interface{} //nolint:deadcode,unused + +// +name:"ElasticSearch GenId" +// +url:"TODO" +// +description:"" +// +status:"" +type _metaElasticsearchGenId interface{} //nolint:deadcode,unused + // +kubebuilder:object:generate=true type ElasticsearchGenId struct { // You can specify generated hash storing key. @@ -39,30 +76,6 @@ type ElasticsearchGenId struct { HashType string `json:"hash_type,omitempty"` } -// ## Example `Elasticsearch Genid` filter configurations -// ```yaml -//apiVersion: logging.banzaicloud.io/v1beta1 -//kind: Flow -//metadata: -// name: demo-flow -//spec: -// filters: -// - elasticsearch_genid: -// hash_id_key: gen_id -// selectors: {} -// localOutputRefs: -// - demo-output -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// @type elasticsearch_genid -// @id test_elasticsearch_genid -// hash_id_key gen_id -// -// ``` - func NewElasticsearchGenId() *ElasticsearchGenId { return &ElasticsearchGenId{} } diff --git a/pkg/sdk/logging/model/filter/enhance_k8s.go b/pkg/sdk/logging/model/filter/enhance_k8s.go index b682ba9a9..05b40d711 100644 --- a/pkg/sdk/logging/model/filter/enhance_k8s.go +++ b/pkg/sdk/logging/model/filter/enhance_k8s.go @@ -37,9 +37,9 @@ type _metaEnhanceK8s interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true type EnhanceK8s struct { - // parameters for read/write record (default: ['$.namespace']) + // parameters for read/write record (default: `['$.namespace']`) InNamespacePath []string `json:"in_namespace_path,omitempty"` - // (default: ['$.pod','$.pod_name']) + // (default: `['$.pod','$.pod_name']`) InPodPath []string `json:"in_pod_path,omitempty"` // Sumologic data type (default: metrics) DataType string `json:"data_type,omitempty"` @@ -59,9 +59,9 @@ type EnhanceK8s struct { VerifySSL *bool `json:"verify_ssl,omitempty"` // Kubernetes core API version (for different Kubernetes versions) (default: ['v1']) CoreAPIVersions []string `json:"core_api_versions,omitempty"` - // Kubernetes resources api groups (default: ["apps/v1", "extensions/v1beta1"]) + // Kubernetes resources api groups (default: `["apps/v1", "extensions/v1beta1"]`) APIGroups []string `json:"api_groups,omitempty"` - // if `ca_file` is for an intermediate CA, or otherwise we do not have the + // If `ca_file` is for an intermediate CA, or otherwise we do not have the // root CA and want to trust the intermediate CA certs we do have, set this // to `true` - this corresponds to the openssl s_client -partial_chain flag // and X509_V_FLAG_PARTIAL_CHAIN (default: false) @@ -76,30 +76,29 @@ type EnhanceK8s struct { CacheRefreshVariation int `json:"cache_refresh_variation,omitempty"` } -// ## Example `EnhanceK8s` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Logging -// metadata: -// -// name: demo-flow -// -// spec: -// -// globalFilters: -// - enhanceK8s: {} -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type enhance_k8s_metadata -// @id test_enhanceK8s -// -// -// ``` +/* +## Example `EnhanceK8s` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: demo-flow +spec: + globalFilters: + - enhanceK8s: {} +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type enhance_k8s_metadata + @id test_enhanceK8s + +{{}} +*/ type _expEnhanceK8s interface{} //nolint:deadcode,unused func (c *EnhanceK8s) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/geoip.go b/pkg/sdk/logging/model/filter/geoip.go index a028d7abd..f2df11753 100644 --- a/pkg/sdk/logging/model/filter/geoip.go +++ b/pkg/sdk/logging/model/filter/geoip.go @@ -51,49 +51,48 @@ type GeoIP struct { Records []Record `json:"records,omitempty"` } -// ## Example `GeoIP` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - geoip: -// geoip_lookup_keys: remote_addr -// records: -// - city: ${city.names.en["remote_addr"]} -// location_array: '''[${location.longitude["remote"]},${location.latitude["remote"]}]''' -// country: ${country.iso_code["remote_addr"]} -// country_name: ${country.names.en["remote_addr"]} -// postal_code: ${postal.code["remote_addr"]} -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type geoip -// @id test_geoip -// geoip_lookup_keys remote_addr -// skip_adding_null_record true -// -// city ${city.names.en["remote_addr"]} -// country ${country.iso_code["remote_addr"]} -// country_name ${country.names.en["remote_addr"]} -// location_array '[${location.longitude["remote"]},${location.latitude["remote"]}]' -// postal_code ${postal.code["remote_addr"]} -// -// -// -// ``` +/* +## Example `GeoIP` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - geoip: + geoip_lookup_keys: remote_addr + records: + - city: ${city.names.en["remote_addr"]} + location_array: '''[${location.longitude["remote"]},${location.latitude["remote"]}]''' + country: ${country.iso_code["remote_addr"]} + country_name: ${country.names.en["remote_addr"]} + postal_code: ${postal.code["remote_addr"]} + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type geoip + @id test_geoip + geoip_lookup_keys remote_addr + skip_adding_null_record true + + city ${city.names.en["remote_addr"]} + country ${country.iso_code["remote_addr"]} + country_name ${country.names.en["remote_addr"]} + location_array '[${location.longitude["remote"]},${location.latitude["remote"]}]' + postal_code ${postal.code["remote_addr"]} + + +{{}} +*/ type _expGeoIP interface{} //nolint:deadcode,unused func (g *GeoIP) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/grep.go b/pkg/sdk/logging/model/filter/grep.go index 07e7b57ea..6082f9a42 100644 --- a/pkg/sdk/logging/model/filter/grep.go +++ b/pkg/sdk/logging/model/filter/grep.go @@ -57,40 +57,39 @@ type RegexpSection struct { Pattern string `json:"pattern"` } -// ## Example `Regexp` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - grep: -// regexp: -// - key: first -// pattern: /^5\d\d$/ -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type grep -// @id demo-flow_1_grep -// -// key first -// pattern /^5\d\d$/ -// -// -// -// ``` +/* +## Example `Regexp` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + regexp: + - key: first + pattern: /^5\d\d$/ + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type grep + @id demo-flow_1_grep + + key first + pattern /^5\d\d$/ + + +{{}} +*/ type _expRegexp interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true @@ -103,40 +102,40 @@ type ExcludeSection struct { Pattern string `json:"pattern"` } -// ## Example `Exclude` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - grep: -// exclude: -// - key: first -// pattern: /^5\d\d$/ -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type grep -// @id demo-flow_0_grep -// -// key first -// pattern /^5\d\d$/ -// -// -// -// ``` +/* +## Example `Exclude` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + exclude: + - key: first + pattern: /^5\d\d$/ + selectors: {} + localOutputRefs: + - demo-output +{{}} + +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type grep + @id demo-flow_0_grep + + key first + pattern /^5\d\d$/ + + +{{}} +*/ type _expExclude interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true @@ -149,46 +148,45 @@ type OrSection struct { Exclude []ExcludeSection `json:"exclude,omitempty"` } -// ## Example `Or` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - grep: -// or: -// - exclude: -// - key: first -// pattern: /^5\d\d$/ -// - key: second -// pattern: /\.css$/ -// -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// -// key first -// pattern /^5\d\d$/ -// -// -// key second -// pattern /\.css$/ -// -// -// -// ``` +/* +## Example `Or` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + or: + - exclude: + - key: first + pattern: /^5\d\d$/ + - key: second + pattern: /\.css$/ + + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + + key first + pattern /^5\d\d$/ + + + key second + pattern /\.css$/ + + +{{}} +*/ type _expOR interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true @@ -201,46 +199,45 @@ type AndSection struct { Exclude []ExcludeSection `json:"exclude,omitempty"` } -// ## Example `And` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - grep: -// and: -// - regexp: -// - key: first -// pattern: /^5\d\d$/ -// - key: second -// pattern: /\.css$/ -// -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// -// key first -// pattern /^5\d\d$/ -// -// -// key second -// pattern /\.css$/ -// -// -// -// ``` +/* +## Example `And` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - grep: + and: + - regexp: + - key: first + pattern: /^5\d\d$/ + - key: second + pattern: /\.css$/ + + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +Fluentd #### config result: + +{{< highlight xml >}} + + + key first + pattern /^5\d\d$/ + + + key second + pattern /\.css$/ + + +{{}} +*/ type _expAND interface{} //nolint:deadcode,unused func (r *RegexpSection) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/kube_events_timestamp.go b/pkg/sdk/logging/model/filter/kube_events_timestamp.go index 098c5246c..40fcc9a42 100644 --- a/pkg/sdk/logging/model/filter/kube_events_timestamp.go +++ b/pkg/sdk/logging/model/filter/kube_events_timestamp.go @@ -43,38 +43,39 @@ type KubeEventsTimestampConfig struct { MappedTimeKey string `json:"mapped_time_key,omitempty"` } -// ## Example `Kubernetes Events Timestamp` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: es-flow -// -// spec: -// -// filters: -// - kube_events_timestamp: -// timestamp_fields: -// - "event.eventTime" -// - "event.lastTimestamp" -// - "event.firstTimestamp" -// mapped_time_key: mytimefield -// selectors: {} -// localOutputRefs: -// - es-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// @type kube_events_timestamp -// @id test-kube-events-timestamp -// timestamp_fields ["event.eventTime","event.lastTimestamp","event.firstTimestamp"] -// mapped_time_key mytimefield -// -// ``` +/* +## Example `Kubernetes Events Timestamp` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: es-flow +spec: + filters: + - kube_events_timestamp: + timestamp_fields: + - "event.eventTime" + - "event.lastTimestamp" + - "event.firstTimestamp" + mapped_time_key: mytimefield + selectors: {} + localOutputRefs: + - es-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type kube_events_timestamp + @id test-kube-events-timestamp + timestamp_fields ["event.eventTime","event.lastTimestamp","event.firstTimestamp"] + mapped_time_key mytimefield + +{{}} +*/ type _expKubeEventsTimestamp interface{} //nolint:deadcode,unused func NewKubeEventsTimestampConfig() *KubeEventsTimestampConfig { diff --git a/pkg/sdk/logging/model/filter/parser.go b/pkg/sdk/logging/model/filter/parser.go index 8384ba432..179b77fa5 100644 --- a/pkg/sdk/logging/model/filter/parser.go +++ b/pkg/sdk/logging/model/filter/parser.go @@ -198,58 +198,57 @@ type GrokSection struct { Timezone string `json:"timezone,omitempty"` } -// ## Example `Parser` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - parser: -// remove_key_name_field: true -// reserve_data: true -// parse: -// type: multi_format -// patterns: -// - format: nginx -// - format: regexp -// expression: /foo/ -// - format: none -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type parser -// @id test_parser -// key_name message -// remove_key_name_field true -// reserve_data true -// -// @type multi_format -// -// format nginx -// -// -// expression /foo/ -// format regexp -// -// -// format none -// -// -// -// -// ``` +/* +## Example `Parser` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - parser: + remove_key_name_field: true + reserve_data: true + parse: + type: multi_format + patterns: + - format: nginx + - format: regexp + expression: /foo/ + - format: none + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight yaml >}} + + @type parser + @id test_parser + key_name message + remove_key_name_field true + reserve_data true + + @type multi_format + + format nginx + + + expression /foo/ + format regexp + + + format none + + + +{{}} +*/ type _expParser interface{} //nolint:deadcode,unused func (p *SingleParseSection) ToPatternDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/prometheus.go b/pkg/sdk/logging/model/filter/prometheus.go index 70954339b..611050ae9 100644 --- a/pkg/sdk/logging/model/filter/prometheus.go +++ b/pkg/sdk/logging/model/filter/prometheus.go @@ -59,62 +59,60 @@ type MetricSection struct { Labels Label `json:"labels,omitempty"` } -// ## Example `Prometheus` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - tag_normaliser: {} -// - parser: -// remove_key_name_field: true -// reserve_data: true -// parse: -// type: nginx -// - prometheus: -// metrics: -// - name: total_counter -// desc: The total number of foo in message. -// type: counter -// labels: -// foo: bar -// labels: -// host: ${hostname} -// tag: ${tag} -// namespace: $.kubernetes.namespace -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ``` -// -// -// @type prometheus -// @id logging-demo-flow_2_prometheus -// -// desc The total number of foo in message. -// name total_counter -// type counter -// -// foo bar -// -// -// -// host ${hostname} -// namespace $.kubernetes.namespace -// tag ${tag} -// -// -// -// ``` +/* +## Example `Prometheus` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - tag_normaliser: {} + - parser: + remove_key_name_field: true + reserve_data: true + parse: + type: nginx + - prometheus: + metrics: + - name: total_counter + desc: The total number of foo in message. + type: counter + labels: + foo: bar + labels: + host: ${hostname} + tag: ${tag} + namespace: $.kubernetes.namespace + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml>}} + + @type prometheus + @id logging-demo-flow_2_prometheus + + desc The total number of foo in message. + name total_counter + type counter + + foo bar + + + + host ${hostname} + namespace $.kubernetes.namespace + tag ${tag} + + +{{}} +*/ type _expPrometheus interface{} //nolint:deadcode,unused type Label map[string]string diff --git a/pkg/sdk/logging/model/filter/record_modifier.go b/pkg/sdk/logging/model/filter/record_modifier.go index 0adfc796d..d10532403 100644 --- a/pkg/sdk/logging/model/filter/record_modifier.go +++ b/pkg/sdk/logging/model/filter/record_modifier.go @@ -37,7 +37,7 @@ type _metaRecordModifier interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true type RecordModifier struct { - // Prepare values for filtering in configure phase. Prepared values can be used in . You can write any ruby code. + // Prepare values for filtering in configure phase. Prepared values can be used in ``. You can write any ruby code. PrepareValues string `json:"prepare_value,omitempty"` // Fluentd including some plugins treats logs as a BINARY by default to forward. To overide that, use a target encoding or a from:to encoding here. CharEncoding string `json:"char_encoding,omitempty"` @@ -52,38 +52,36 @@ type RecordModifier struct { Records []Record `json:"records,omitempty"` } -// ## Example `Record Modifier` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - record_modifier: -// records: -// - foo: "bar" -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type record_modifier -// @id test_record_modifier -// -// foo bar -// -// -// -// ``` +/* +## Example `Record Modifier` filter configurations +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - record_modifier: + records: + - foo: "bar" + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type record_modifier + @id test_record_modifier + + foo bar + + +{{}} +*/ type _expRecordModifier interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true diff --git a/pkg/sdk/logging/model/filter/record_transformer.go b/pkg/sdk/logging/model/filter/record_transformer.go index a1b32fe74..5112895a9 100644 --- a/pkg/sdk/logging/model/filter/record_transformer.go +++ b/pkg/sdk/logging/model/filter/record_transformer.go @@ -45,7 +45,7 @@ type RecordTransformer struct { RenewRecord bool `json:"renew_record,omitempty"` // Specify field name of the record to overwrite the time of events. Its value must be unix time. RenewTimeKey string `json:"renew_time_key,omitempty"` - // When set to true, the full Ruby syntax is enabled in the ${...} expression. (default: false) + // When set to true, the full Ruby syntax is enabled in the `${...}` expression. (default: false) EnableRuby bool `json:"enable_ruby,omitempty"` // Use original value type. (default: true) AutoTypecast bool `json:"auto_typecast,omitempty"` @@ -54,38 +54,38 @@ type RecordTransformer struct { Records []Record `json:"records,omitempty"` } -// ## Example `Record Transformer` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - record_transformer: -// records: -// - foo: "bar" -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type record_transformer -// @id test_record_transformer -// -// foo bar -// -// -// -// ``` +/* +## Example `Record Transformer` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - record_transformer: + records: + - foo: "bar" + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* + +Fluentd config result: + +{{< highlight xml >}} + + @type record_transformer + @id test_record_transformer + + foo bar + + +{{}} +*/ type _expRecordTransformer interface{} //nolint:deadcode,unused // Parameters inside record directives are considered to be new key-value pairs diff --git a/pkg/sdk/logging/model/filter/stdout.go b/pkg/sdk/logging/model/filter/stdout.go index a8bf5abf6..d35f1a752 100644 --- a/pkg/sdk/logging/model/filter/stdout.go +++ b/pkg/sdk/logging/model/filter/stdout.go @@ -41,35 +41,34 @@ type StdOutFilterConfig struct { OutputType string `json:"output_type,omitempty"` } -// ## Example `StdOut` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - stdout: -// output_type: json -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type stdout -// @id test_stdout -// output_type json -// -// -// ``` +/* +## Example `StdOut` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - stdout: + output_type: json + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight yaml >}} + + @type stdout + @id test_stdout + output_type json + +{{}} +*/ type _expStdOut interface{} //nolint:deadcode,unused func NewStdOutFilterConfig() *StdOutFilterConfig { diff --git a/pkg/sdk/logging/model/filter/sumologic.go b/pkg/sdk/logging/model/filter/sumologic.go index 262c0dc8f..11347db65 100644 --- a/pkg/sdk/logging/model/filter/sumologic.go +++ b/pkg/sdk/logging/model/filter/sumologic.go @@ -36,13 +36,13 @@ type _metaSumologic interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true type SumoLogic struct { - // Source Category (default: "%{namespace}/%{pod_name}") + // Source Category (default: `%{namespace}/%{pod_name}`) SourceCategory string `json:"source_category,omitempty"` // Source Category Replace Dash (default: "/") SourceCategoryReplaceDash string `json:"source_category_replace_dash,omitempty"` // Source Category Prefix (default: kubernetes/) SourceCategoryPrefix string `json:"source_category_prefix,omitempty"` - // Source Name (default: "%{namespace}.%{pod}.%{container}") + // Source Name (default: `%{namespace}.%{pod}.%{container}`) SourceName string `json:"source_name,omitempty"` // Log Format (default: json) LogFormat string `json:"log_format,omitempty"` @@ -74,51 +74,50 @@ type SumoLogic struct { TracingContainerName string `json:"tracing_container_name,omitempty"` // Tracing Host (default: "hostname") TracingHost string `json:"tracing_host,omitempty"` - // Tracing Label Prefix (default: "pod_label_") + // Tracing Label Prefix (default: `pod_label_`) TracingLabelPrefix string `json:"tracing_label_prefix,omitempty"` - // Tracing Annotation Prefix (default: "pod_annotation_") + // Tracing Annotation Prefix (default: `pod_annotation_`) TracingAnnotationPrefix string `json:"tracing_annotation_prefix,omitempty"` - // Source HostKey Name (default: "_sourceHost") + // Source HostKey Name (default: `_sourceHost`) SourceHostKeyName string `json:"source_host_key_name,omitempty"` - // Source CategoryKey Name (default: "_sourceCategory") + // Source CategoryKey Name (default: `_sourceCategory`) SourceCategoryKeyName string `json:"source_category_key_name,omitempty"` - // Source NameKey Name (default: "_sourceName") + // Source NameKey Name (default: `_sourceName`) SourceNameKeyName string `json:"source_name_key_name,omitempty"` - // CollectorKey Name (default: "_collector") + // CollectorKey Name (default: `_collector`) CollectorKeyName string `json:"collector_key_name,omitempty"` // Collector Value (default: "undefined") CollectorValue string `json:"collector_value,omitempty"` } -// ## Example `Parser` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - sumologic: -// source_name: "elso" -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type kubernetes_sumologic -// @id test_sumologic -// source_name elso -// -// -// ``` +/* +## Example `Parser` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - sumologic: + source_name: "elso" + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type kubernetes_sumologic + @id test_sumologic + source_name elso + +{{}} +*/ type _expSumologic interface{} //nolint:deadcode,unused func (s *SumoLogic) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/tagnormaliser.go b/pkg/sdk/logging/model/filter/tagnormaliser.go index 5d4e232c1..ca262338c 100644 --- a/pkg/sdk/logging/model/filter/tagnormaliser.go +++ b/pkg/sdk/logging/model/filter/tagnormaliser.go @@ -24,19 +24,22 @@ import ( type _hugoTagNormaliser interface{} //nolint:deadcode,unused // +docName:"Fluentd Plugin to re-tag based on log metadata" -// More info at https://github.com/kube-logging/fluent-plugin-tag-normaliser -// -// # Available kubernetes metadata -// -// | Parameter | Description | Example | -// |-----------|-------------|---------| -// | ${pod_name} | Pod name | understood-butterfly-logging-demo-7dcdcfdcd7-h7p9n | -// | ${container_name} | Container name inside the Pod | logging-demo | -// | ${namespace_name} | Namespace name | default | -// | ${pod_id} | Kubernetes UUID for Pod | 1f50d309-45a6-11e9-b795-025000000001 | -// | ${labels} | Kubernetes Pod labels. This is a nested map. You can access nested attributes via `.` | {"app":"logging-demo", "pod-template-hash":"7dcdcfdcd7" } | -// | ${host} | Node hostname the Pod runs on | docker-desktop | -// | ${docker_id} | Docker UUID of the container | 3a38148aa37aa3... | +/* +More info at https://github.com/kube-logging/fluent-plugin-tag-normaliser + +## Available Kubernetes metadata + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `${pod_name}` | Pod name | understood-butterfly-logging-demo-7dcdcfdcd7-h7p9n | +| `${container_name}` | Container name inside the Pod | logging-demo | +| `${namespace_name}` | Namespace name | default | +| `${pod_id}` | Kubernetes UUID for Pod | 1f50d309-45a6-11e9-b795-025000000001 | +| `${labels}` | Kubernetes Pod labels. This is a nested map. You can access nested attributes via `.` | `{"app":"logging-demo", "pod-template-hash":"7dcdcfdcd7" }` | +| `${host}` | Node hostname the Pod runs on | docker-desktop | +| `${docker_id}` | Docker UUID of the container | 3a38148aa37aa3... | + +*/ type _docTagNormaliser interface{} //nolint:deadcode,unused // +name:"Tag Normaliser" @@ -50,39 +53,38 @@ type _metaTagNormaliser interface{} //nolint:deadcode,unused type TagNormaliser struct { // Re-Tag log messages info at [github](https://github.com/kube-logging/fluent-plugin-tag-normaliser) Format string `json:"format,omitempty" plugin:"default:${namespace_name}.${pod_name}.${container_name}"` - // Tag used in match directive. (default: kubernetes.**) + // Tag used in match directive. (default: `kubernetes.**`) MatchTag string `json:"match_tag,omitempty" plugin:"hidden"` } -// ## Example `Parser` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - tag_normaliser: -// format: cluster1.${namespace_name}.${pod_name}.${labels.app} -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type tag_normaliser -// @id test_tag_normaliser -// format cluster1.${namespace_name}.${pod_name}.${labels.app} -// -// -// ``` +/* +## Example `Parser` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - tag_normaliser: + format: cluster1.${namespace_name}.${pod_name}.${labels.app} + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type tag_normaliser + @id test_tag_normaliser + format cluster1.${namespace_name}.${pod_name}.${labels.app} + +{{}} +*/ type _expTagNormaliser interface{} //nolint:deadcode,unused func (t *TagNormaliser) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/filter/throttle.go b/pkg/sdk/logging/model/filter/throttle.go index 7268ecb6a..48ce0dce5 100644 --- a/pkg/sdk/logging/model/filter/throttle.go +++ b/pkg/sdk/logging/model/filter/throttle.go @@ -19,6 +19,10 @@ import ( "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types" ) +// +name:"Throttle" +// +weight:"200" +type _hugoThrottle interface{} //nolint:deadcode,unused + // +kubebuilder:object:generate=true // +docName:"[Throttle Filter](https://github.com/rubrikinc/fluent-plugin-throttle)" // A sentry plugin to throttle logs. Logs are grouped by a configurable key. When a group exceeds a configuration rate, logs are dropped for this group. @@ -47,35 +51,34 @@ type Throttle struct { GroupWarningDelaySeconds int `json:"group_warning_delay_s,omitempty"` } -// ## Example `Throttle` filter configurations -// ```yaml -// apiVersion: logging.banzaicloud.io/v1beta1 -// kind: Flow -// metadata: -// -// name: demo-flow -// -// spec: -// -// filters: -// - throttle: -// group_key: "$.kubernetes.container_name" -// selectors: {} -// localOutputRefs: -// - demo-output -// -// ``` -// -// #### Fluentd Config Result -// ```yaml -// -// -// @type throttle -// @id test_throttle -// group_key $.kubernetes.container_name -// -// -// ``` +/* +## Example `Throttle` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - throttle: + group_key: "$.kubernetes.container_name" + selectors: {} + localOutputRefs: + - demo-output +{{}} +*/ +/* +#### Fluentd config result: + +{{< highlight xml >}} + + @type throttle + @id test_throttle + group_key $.kubernetes.container_name + +{{}} +*/ type _expThrottle interface{} //nolint:deadcode,unused func (t *Throttle) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { diff --git a/pkg/sdk/logging/model/syslogng/filter/match.go b/pkg/sdk/logging/model/syslogng/filter/match.go index 995157e01..efd8bf23f 100644 --- a/pkg/sdk/logging/model/syslogng/filter/match.go +++ b/pkg/sdk/logging/model/syslogng/filter/match.go @@ -20,21 +20,23 @@ type _hugoMatch interface{} //nolint:deadcode,unused // +kubebuilder:object:generate=true // +docName:"Match" -/* Match filters can be used to select the log records to process. These filters have the same options and syntax as the [syslog-ng flow match expressions]({{< relref "/docs/configuration/plugins/syslog-ng-filters/match.md" >}}). - -{{< highlight yaml >}} - filters: - - match: - or: - - regexp: - value: json.kubernetes.labels.app.kubernetes.io/name - pattern: apache - type: string - - regexp: - value: json.kubernetes.labels.app.kubernetes.io/name - pattern: nginx - type: string -{{}} */ +/* Match filters can be used to select the log records to process. These filters have the same options and syntax as [syslog-ng flow match expressions]({{< relref "/docs/configuration/plugins/syslogng-filters/match.md" >}}). +// +// {{< highlight yaml >}} +// +// filters: +// - match: +// or: +// - regexp: +// value: json.kubernetes.labels.app.kubernetes.io/name +// pattern: apache +// type: string +// - regexp: +// value: json.kubernetes.labels.app.kubernetes.io/name +// pattern: nginx +// type: string +// +// {{}} */ type _docMatch interface{} //nolint:deadcode,unused // +name:"Syslog-NG Match"