From ef0d2528a087dee7336081290a9875075d06c4e2 Mon Sep 17 00:00:00 2001 From: braydonk Date: Tue, 5 Dec 2023 16:56:56 +0000 Subject: [PATCH 1/3] stackdriver: amend special fields docs Signed-off-by: braydonk --- .../outputs/stackdriver_special_fields.md | 74 +++++++++++-------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/pipeline/outputs/stackdriver_special_fields.md b/pipeline/outputs/stackdriver_special_fields.md index 0d15201a5..eff5d0ee4 100644 --- a/pipeline/outputs/stackdriver_special_fields.md +++ b/pipeline/outputs/stackdriver_special_fields.md @@ -1,9 +1,11 @@ # Stackdriver Special fields -When the [google-logging-agent](https://cloud.google.com/logging/docs/agent) receives a structured log record, it treats the [some fields](https://cloud.google.com/logging/docs/agent/configuration#special-fields) specially, allowing users to set specific fields in the LogEntry object that get written to the Logging API. +When the [google-logging-agent](https://cloud.google.com/logging/docs/agent) receives a structured log record, it treats [some fields](https://cloud.google.com/logging/docs/agent/configuration#special-fields) specially, allowing users to set specific fields in the LogEntry object that get written to the Logging API. -Currently, we also support some special fields in fluent-bit: -| JSON log field | [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) field | Logging agent function | +## Log Entry Fields + +Currently, we support some special fields in fluent-bit for setting fields on the LogEntry object: +| JSON log field | [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) field | Description | | :--- | :--- | :--- | | logging.googleapis.com/operation | operation | Additional information about a potentially long-running operation | | logging.googleapis.com/labels | labels | The value of this field should be a structured record | @@ -11,9 +13,17 @@ Currently, we also support some special fields in fluent-bit: | logging.googleapis.com/sourceLocation | sourceLocation | Additional information about the source code location that produced the log entry. | | logging.googleapis.com/http_request | httpRequest | A common proto for logging HTTP requests. | | logging.googleapis.com/trace | trace | Resource name of the trace associated with the log entry | +| logging.googleapis.com/traceSampled | traceSampled | The sampling decision associated with this log entry. | +| logging.googleapis.com/spanId | spanId | The ID of the trace span associated with this log entry. | | timestamp | timestamp | An object including the seconds and nanos fields that represents the time | | timestampSecond & timestampNanos | timestamp | The seconds and nanos that represents the time | +## Other Special Fields + +| JSON log field | Description | +| :--- | :--- | +| logging.googleapis.com/projectId | Changes the project ID that this log will be written to. Ensure that you are authenticated to write logs to this project. | + ## Operation Operation field contains additional information about a potentially long-running operation with which a log entry is associated. @@ -239,7 +249,7 @@ the logEntry will be: ``` ## httpRequest -HttpRequest field is a common proto for logging HTTP requests. +HttpRequest field is a common proto for logging HTTP requests. The JSON representation is as followed: ```text @@ -268,20 +278,20 @@ Set the input log as followed: ```text jsonPayload { "logging.googleapis.com/http_request": { - "requestMethod":"GET", - "requestUrl":"logging.googleapis.com", - "requestSize":"12", - "status":200, - "responseSize":"12", - "userAgent":"Mozilla", - "remoteIp":"255.0.0.1", - "serverIp":"255.0.0.1", - "referer":"referer", - "latency":"1s", - "cacheLookup":true, - "cacheHit":true, - "cacheValidatedWithOriginServer":true, - "cacheFillBytes":"12", + "requestMethod":"GET", + "requestUrl":"logging.googleapis.com", + "requestSize":"12", + "status":200, + "responseSize":"12", + "userAgent":"Mozilla", + "remoteIp":"255.0.0.1", + "serverIp":"255.0.0.1", + "referer":"referer", + "latency":"1s", + "cacheLookup":true, + "cacheHit":true, + "cacheValidatedWithOriginServer":true, + "cacheFillBytes":"12", "protocol":"HTTP/1.2" } ... @@ -294,20 +304,20 @@ the logEntry will be: ... } "httpRequest": { - "requestMethod":"GET", - "requestUrl":"logging.googleapis.com", - "requestSize":"12", - "status":200, - "responseSize":"12", - "userAgent":"Mozilla", - "remoteIp":"255.0.0.1", - "serverIp":"255.0.0.1", - "referer":"referer", - "latency":"1s", - "cacheLookup":true, - "cacheHit":true, - "cacheValidatedWithOriginServer":true, - "cacheFillBytes":"12", + "requestMethod":"GET", + "requestUrl":"logging.googleapis.com", + "requestSize":"12", + "status":200, + "responseSize":"12", + "userAgent":"Mozilla", + "remoteIp":"255.0.0.1", + "serverIp":"255.0.0.1", + "referer":"referer", + "latency":"1s", + "cacheLookup":true, + "cacheHit":true, + "cacheValidatedWithOriginServer":true, + "cacheFillBytes":"12", "protocol":"HTTP/1.2" } ... From 1c37809a03386629028cadf4d133ae83bd8af969 Mon Sep 17 00:00:00 2001 From: braydonk Date: Wed, 10 Jan 2024 20:38:22 +0000 Subject: [PATCH 2/3] add project_id_key to stackdriver docs and link out to special fields docs for key fields Signed-off-by: braydonk --- pipeline/outputs/stackdriver.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pipeline/outputs/stackdriver.md b/pipeline/outputs/stackdriver.md index add3b5bd8..ae635d51a 100644 --- a/pipeline/outputs/stackdriver.md +++ b/pipeline/outputs/stackdriver.md @@ -25,11 +25,12 @@ Before to get started with the plugin configuration, make sure to obtain the pro | resource | Set resource type of data. Supported resource types: _k8s\_container_, _k8s\_node_, _k8s\_pod_, _global_, _generic\_node_, _generic\_task_, and _gce\_instance_. | global, gce\_instance | | k8s\_cluster\_name | The name of the cluster that the container \(node or pod based on the resource type\) is running in. If the resource type is one of the _k8s\_container_, _k8s\_node_ or _k8s\_pod_, then this field is required. | | | k8s\_cluster\_location | The physical location of the cluster that contains \(node or pod based on the resource type\) the container. If the resource type is one of the _k8s\_container_, _k8s\_node_ or _k8s\_pod_, then this field is required. | | -| labels\_key | The value of this field is used by the Stackdriver output plugin to find the related labels from jsonPayload and then extract the value of it to set the LogEntry Labels. | logging.googleapis.com/labels | +| labels\_key | The value of this field is used by the Stackdriver output plugin to find the related labels from jsonPayload and then extract the value of it to set the LogEntry Labels. | `logging.googleapis.com/labels`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | | labels | Optional list of comma separated of strings specifying `key=value` pairs. The resulting `labels` will be combined with the elements in obtained from `labels_key` to set the LogEntry Labels. Elements from `labels` will override duplicate values from `labels_key`.| | -| log\_name\_key | The value of this field is used by the Stackdriver output plugin to extract logName from jsonPayload and set the logName field. | logging.googleapis.com/logName | +| log\_name\_key | The value of this field is used by the Stackdriver output plugin to extract logName from jsonPayload and set the logName field. | `logging.googleapis.com/logName`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | | tag\_prefix | Set the tag\_prefix used to validate the tag of logs with k8s resource type. Without this option, the tag of the log must be in format of k8s\_container\(pod/node\).\* in order to use the k8s\_container resource type. Now the tag prefix is configurable by this option \(note the ending dot\). | k8s\_container., k8s\_pod., k8s\_node. | -| severity\_key | Specify the name of the key from the original record that contains the severity information. | | +| severity\_key | Specify the name of the key from the original record that contains the severity information. | `logging.googleapis.com/severity`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | +| project_id_key | The value of this field is used by the Stackdriver output plugin to find the gcp project id from jsonPayload and then extract the value of it to set the PROJECT_ID within LogEntry logName, which controls the gcp project that should receive these logs. | `logging.googleapis.com/projectId`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | | autoformat\_stackdriver\_trace | Rewrite the _trace_ field to include the projectID and format it for use with Cloud Trace. When this flag is enabled, the user can get the correct result by printing only the traceID (usually 32 characters). | false | | Workers | Enables dedicated thread(s) for this output. | 1 | | custom\_k8s\_regex | Set a custom regex to extract field like pod\_name, namespace\_name, container\_name and docker\_id from the local\_resource\_id in logs. This is helpful if the value of pod or node name contains dots. | `(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$` | @@ -140,7 +141,7 @@ This will produce the following log: } ``` -This makes the `resource_labels` API the recommended choice for supporting new or existing resource types that have all resource labels known before runtime or available on the payload during runtime. +This makes the `resource_labels` API the recommended choice for supporting new or existing resource types that have all resource labels known before runtime or available on the payload during runtime. For instance, for a K8s resource type, `resource_labels` can be used in tandem with the [Kubernetes filter](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes) to pack all six resource labels. Below is an example of what this could look like for a `k8s_container` resource: @@ -189,3 +190,5 @@ Do following check: Stackdriver officially supports a [logging agent based on Fluentd](https://cloud.google.com/logging/docs/agent). We plan to support some [special fields in structured payloads](https://cloud.google.com/logging/docs/agent/configuration#special-fields). Use cases of special fields is [here](./stackdriver_special_fields.md). + +[StackdriverSpecialFields]: ./stackdriver_special_fields.md#log-entry-fields \ No newline at end of file From 528282fb88b5dffd47abd71583ce1d7e2bb7ab6f Mon Sep 17 00:00:00 2001 From: braydonk Date: Thu, 22 Feb 2024 14:41:00 +0000 Subject: [PATCH 3/3] fix broken link syntax Signed-off-by: braydonk --- pipeline/outputs/stackdriver.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline/outputs/stackdriver.md b/pipeline/outputs/stackdriver.md index ae635d51a..fe4894b86 100644 --- a/pipeline/outputs/stackdriver.md +++ b/pipeline/outputs/stackdriver.md @@ -25,12 +25,12 @@ Before to get started with the plugin configuration, make sure to obtain the pro | resource | Set resource type of data. Supported resource types: _k8s\_container_, _k8s\_node_, _k8s\_pod_, _global_, _generic\_node_, _generic\_task_, and _gce\_instance_. | global, gce\_instance | | k8s\_cluster\_name | The name of the cluster that the container \(node or pod based on the resource type\) is running in. If the resource type is one of the _k8s\_container_, _k8s\_node_ or _k8s\_pod_, then this field is required. | | | k8s\_cluster\_location | The physical location of the cluster that contains \(node or pod based on the resource type\) the container. If the resource type is one of the _k8s\_container_, _k8s\_node_ or _k8s\_pod_, then this field is required. | | -| labels\_key | The value of this field is used by the Stackdriver output plugin to find the related labels from jsonPayload and then extract the value of it to set the LogEntry Labels. | `logging.googleapis.com/labels`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | +| labels\_key | The value of this field is used by the Stackdriver output plugin to find the related labels from jsonPayload and then extract the value of it to set the LogEntry Labels. | `logging.googleapis.com/labels`. See [Stackdriver Special Fields][StackdriverSpecialFields] for more info. | | labels | Optional list of comma separated of strings specifying `key=value` pairs. The resulting `labels` will be combined with the elements in obtained from `labels_key` to set the LogEntry Labels. Elements from `labels` will override duplicate values from `labels_key`.| | -| log\_name\_key | The value of this field is used by the Stackdriver output plugin to extract logName from jsonPayload and set the logName field. | `logging.googleapis.com/logName`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | +| log\_name\_key | The value of this field is used by the Stackdriver output plugin to extract logName from jsonPayload and set the logName field. | `logging.googleapis.com/logName`. See [Stackdriver Special Fields][StackdriverSpecialFields] for more info. | | tag\_prefix | Set the tag\_prefix used to validate the tag of logs with k8s resource type. Without this option, the tag of the log must be in format of k8s\_container\(pod/node\).\* in order to use the k8s\_container resource type. Now the tag prefix is configurable by this option \(note the ending dot\). | k8s\_container., k8s\_pod., k8s\_node. | -| severity\_key | Specify the name of the key from the original record that contains the severity information. | `logging.googleapis.com/severity`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | -| project_id_key | The value of this field is used by the Stackdriver output plugin to find the gcp project id from jsonPayload and then extract the value of it to set the PROJECT_ID within LogEntry logName, which controls the gcp project that should receive these logs. | `logging.googleapis.com/projectId`. See [Stackdriver Special Fields](StackdriverSpecialFields) for more info. | +| severity\_key | Specify the name of the key from the original record that contains the severity information. | `logging.googleapis.com/severity`. See [Stackdriver Special Fields][StackdriverSpecialFields] for more info. | +| project_id_key | The value of this field is used by the Stackdriver output plugin to find the gcp project id from jsonPayload and then extract the value of it to set the PROJECT_ID within LogEntry logName, which controls the gcp project that should receive these logs. | `logging.googleapis.com/projectId`. See [Stackdriver Special Fields][StackdriverSpecialFields] for more info. | | autoformat\_stackdriver\_trace | Rewrite the _trace_ field to include the projectID and format it for use with Cloud Trace. When this flag is enabled, the user can get the correct result by printing only the traceID (usually 32 characters). | false | | Workers | Enables dedicated thread(s) for this output. | 1 | | custom\_k8s\_regex | Set a custom regex to extract field like pod\_name, namespace\_name, container\_name and docker\_id from the local\_resource\_id in logs. This is helpful if the value of pod or node name contains dots. | `(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$` | @@ -191,4 +191,4 @@ Stackdriver officially supports a [logging agent based on Fluentd](https://cloud We plan to support some [special fields in structured payloads](https://cloud.google.com/logging/docs/agent/configuration#special-fields). Use cases of special fields is [here](./stackdriver_special_fields.md). -[StackdriverSpecialFields]: ./stackdriver_special_fields.md#log-entry-fields \ No newline at end of file +[StackdriverSpecialFields]: ./stackdriver_special_fields.md#log-entry-fields