diff --git a/config/config-observability.yaml b/config/config-observability.yaml index 063e1219795e..d3f88dd9d3f4 100644 --- a/config/config-observability.yaml +++ b/config/config-observability.yaml @@ -31,6 +31,24 @@ data: # The fluentd sidecar output config to specify logging destination. logging.fluentd-sidecar-output-config: | + # Parse json log before sending to Elastic Search + + @type parser + key_name log + + @type multi_format + + format json + time_key fluentd-time # fluentd-time is reserved for structured logs + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format none + message_key log + + + + # Send to Elastic Search @id elasticsearch @type elasticsearch diff --git a/config/monitoring/150-elasticsearch-dev/100-fluentd-configmap.yaml b/config/monitoring/150-elasticsearch-dev/100-fluentd-configmap.yaml index 069ed8ccafdc..3376a9615e91 100644 --- a/config/monitoring/150-elasticsearch-dev/100-fluentd-configmap.yaml +++ b/config/monitoring/150-elasticsearch-dev/100-fluentd-configmap.yaml @@ -49,6 +49,8 @@ data: # Add Kubernetes metadata @type kubernetes_metadata + merge_json_log false # Don't parse json log + preserve_json_log false 300.forward.input.conf: |- # Takes the messages sent over TCP, e.g. request logs from Istio @@ -57,6 +59,23 @@ data: port 24224 900.output.conf: |- + # Parse json log before sending to Elastic Search + + @type parser + key_name log + + @type multi_format + + format json + time_key fluentd-time # fluentd-time is reserved for structured logs + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format none + message_key log + + + # Send to Elastic Search @id elasticsearch diff --git a/config/monitoring/150-elasticsearch-prod/100-fluentd-configmap.yaml b/config/monitoring/150-elasticsearch-prod/100-fluentd-configmap.yaml index 031e864d47ef..1b936b8d4455 100644 --- a/config/monitoring/150-elasticsearch-prod/100-fluentd-configmap.yaml +++ b/config/monitoring/150-elasticsearch-prod/100-fluentd-configmap.yaml @@ -49,6 +49,8 @@ data: # Add Kubernetes metadata @type kubernetes_metadata + merge_json_log false # Don't parse json log + preserve_json_log false 300.forward.input.conf: |- # Takes the messages sent over TCP, e.g. request logs from Istio @@ -57,6 +59,23 @@ data: port 24224 900.output.conf: |- + # Parse json log before sending to Elastic Search + + @type parser + key_name log + + @type multi_format + + format json + time_key fluentd-time # fluentd-time is reserved for structured logs + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format none + message_key log + + + # Send to Elastic Search @id elasticsearch diff --git a/config/monitoring/150-stackdriver-dev/fluentd-configmap.yaml b/config/monitoring/150-stackdriver-dev/fluentd-configmap.yaml index 87aaef3eb76c..32b17bedb43d 100644 --- a/config/monitoring/150-stackdriver-dev/fluentd-configmap.yaml +++ b/config/monitoring/150-stackdriver-dev/fluentd-configmap.yaml @@ -50,6 +50,8 @@ data: # Add Kubernetes metadata @type kubernetes_metadata + merge_json_log false # Don't parse json log + preserve_json_log false 300.forward.input.conf: |- # Takes the messages sent over TCP, e.g. request logs from Istio diff --git a/config/monitoring/150-stackdriver-prod/fluentd-configmap.yaml b/config/monitoring/150-stackdriver-prod/fluentd-configmap.yaml index 7b7b9bd3893a..26c84303eeb0 100644 --- a/config/monitoring/150-stackdriver-prod/fluentd-configmap.yaml +++ b/config/monitoring/150-stackdriver-prod/fluentd-configmap.yaml @@ -50,6 +50,8 @@ data: # Add Kubernetes metadata @type kubernetes_metadata + merge_json_log false # Don't parse json log + preserve_json_log false 300.forward.input.conf: |- # Takes the messages sent over TCP, e.g. request logs from Istio diff --git a/image/fluentd/README.md b/image/fluentd/README.md index f72e90553658..7d44a9d1d61e 100644 --- a/image/fluentd/README.md +++ b/image/fluentd/README.md @@ -9,7 +9,7 @@ define logging output. Knative requires the following Fluentd plugins to process log records: * [fluentd](https://github.com/fluent/fluentd) >= v0.14.0 -* [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) >= 1.0.0 +* [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) >= 1.0.0 AND < 2.1.0 * [fluent-plugin-detect-exceptions](https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions) >= 0.0.9 * [fluent-plugin-multi-format-parser](https://github.com/repeatedly/fluent-plugin-multi-format-parser) >= 1.0.0 diff --git a/pkg/controller/revision/fluentd.go b/pkg/controller/revision/fluentd.go index 1954f230767f..51b4f0ef8961 100644 --- a/pkg/controller/revision/fluentd.go +++ b/pkg/controller/revision/fluentd.go @@ -29,18 +29,8 @@ const fluentdSidecarPreOutputConfig = ` path /var/log/revisions/**/*.* pos_file /var/log/varlog.log.pos tag raw.* - - @type multi_format - - format json - time_key fluentd-time # fluentd-time is reserved for structured logs - time_format %Y-%m-%dT%H:%M:%S.%NZ - - - format none - message_key log - - + format none + message_key log read_from_head true diff --git a/sample/telemetrysample/README.md b/sample/telemetrysample/README.md index 099d7c872c0c..0697b271e6ea 100644 --- a/sample/telemetrysample/README.md +++ b/sample/telemetrysample/README.md @@ -10,7 +10,8 @@ that is installed by default as a showcase of installing dedicated Prometheus in ## Prerequisites 1. [Install Knative Serving](https://github.com/knative/install/blob/master/README.md) -1. Install [docker](https://www.docker.com/) +2. [Install Knative monitoring component](docs/telemetry.md) +3. Install [docker](https://www.docker.com/) ## Setup diff --git a/sample/telemetrysample/telemetrysample.go b/sample/telemetrysample/telemetrysample.go index 90f8cee8451a..f8d3d61ad4f4 100644 --- a/sample/telemetrysample/telemetrysample.go +++ b/sample/telemetrysample/telemetrysample.go @@ -230,7 +230,15 @@ func logHandler(client *http.Client) http.HandlerFunc { data := map[string]string{ "log": "A log in json format to STDOUT", + "foo": "bar", "time": timestamp.String(), + // Cluster operator can configure which field is used as time key and what + // the format is. For example, in config/monitoring/150-elasticsearch-dev/100-fluentd-configmap.yaml, + // fluentd-time is the reserved key to tell fluentd the logging time. It + // must be in the format of RFC3339Nano, i.e. %Y-%m-%dT%H:%M:%S.%NZ. + // Without this, fluentd uses the time when it collect the log as an + // event time. + "fluentd-time": timestamp.Format(time.RFC3339Nano), } jsonOutput, _ := json.Marshal(data) fmt.Fprintln(os.Stdout, string(jsonOutput)) @@ -246,15 +254,7 @@ func logHandler(client *http.Client) http.HandlerFunc { fmt.Fprintf(os.Stderr, "Failed to write to %s: %v", fileName, err) } - data := map[string]string{ - "log": "A log in json format to /var/log", - "time": timestamp.String(), - // fluentd-time is the reserved key to tell fluentd the logging time. It - // must be in the format of RFC3339Nano, i.e. %Y-%m-%dT%H:%M:%S.%NZ. - // Without this, fluentd uses current time when it collect the log as an - // event time. - "fluentd-time": timestamp.Format(time.RFC3339Nano), - } + data["log"] = "A log in json format to /var/log" jsonOutput, _ := json.Marshal(data) if _, err := f.WriteString(string(jsonOutput) + "\n"); err != nil { fmt.Fprintf(os.Stderr, "Failed to write to %s: %v", fileName, err)