Add workload fallback logics for entity service name (fluent#18)

amazon-contributing · Oct 1, 2024 · 247a36f · 247a36f
1 parent 85866b1
commit 247a36f
Show file tree

Hide file tree

Showing 9 changed files with 269 additions and 0 deletions.
diff --git a/plugins/filter_kubernetes/kube_meta.c b/plugins/filter_kubernetes/kube_meta.c
@@ -737,12 +737,24 @@ static void cb_results_workload(const char *name, const char *value,
     }
 }
 
+/*
+ * Search workload based on the following priority
+ * where the top is highest priority
+ * 1. Deployment name
+ * 2. StatefulSet name
+ * 3. DaemonSet name
+ * 4. Job name
+ * 5. CronJob name
+ * 6. Pod name
+ * 7. Container name
+ */
 static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgpack_object map)
 {
     int i,j,ownerIndex;
     int regex_found;
     int replicaset_match;
     int podname_match = FLB_FALSE;
+    int workload_found = FLB_FALSE;
     msgpack_object k, v;
     msgpack_object_map ownerMap;
     struct flb_regex_search result;
@@ -788,6 +800,7 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp
                             if (strncmp(key.via.str.ptr, "name", key.via.str.size) == 0) {
                                 /* Store the value of 'name' in workload_val so it can be reused by set_workload */
                                 workload_val = val;
+                                workload_found = FLB_TRUE;
                                 if (replicaset_match) {
                                     regex_found = flb_regex_do(ctx->deploymentRegex, val.via.str.ptr, val.via.str.size, &result);
                                     if (regex_found > 0) {
@@ -808,6 +821,17 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp
             }
         }
     }
+    if(!workload_found) {
+        if(meta->podname != NULL) {
+            meta->workload = flb_strndup(meta->podname, meta->podname_len);
+            meta->workload_len = meta->podname_len;
+            meta->fields++;
+        } else if (meta->container_name != NULL) {
+            meta->workload = flb_strndup(meta->container_name, meta->container_name_len);
+            meta->workload_len = meta->container_name_len;
+            meta->fields++;
+        }
+    }
 
 return;
 

diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c
@@ -969,6 +969,10 @@ void parse_entity(struct flb_cloudwatch *ctx, entity *entity, msgpack_object map
             }
         }
     }
+    if(entity->key_attributes->name == NULL && entity->attributes->name_source == NULL &&entity->attributes->workload != NULL) {
+        entity->key_attributes->name = flb_strndup(entity->attributes->workload, strlen(entity->attributes->workload));
+        entity->attributes->name_source = flb_strndup("K8sWorkload", 11);
+    }
 }
 
 void update_or_create_entity(struct flb_cloudwatch *ctx, struct log_stream *stream, const msgpack_object map) {

diff --git a/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log
@@ -0,0 +1 @@
+{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"}
diff --git a/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log
@@ -0,0 +1 @@
+{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"}
diff --git a/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta b/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta
@@ -0,0 +1,116 @@
+{
+    "apiVersion": "v1",
+    "kind": "Pod",
+    "metadata": {
+        "annotations": {
+            "prometheus.io/path": "/api/v1/metrics/prometheus",
+            "prometheus.io/port": "2020",
+            "prometheus.io/scrape": "true"
+        },
+        "creationTimestamp": "2019-04-03T09:29:00Z",
+        "labels": {
+            "app.kubernetes.io/name": "fluent-bit"
+        },
+        "name": "use-kubelet-disabled-pod",
+        "namespace": "options",
+        "resourceVersion": "74466568",
+        "selfLink": "/api/v1/namespaces/core/pods/base",
+        "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a"
+    },
+    "spec": {
+        "containers": [
+            {
+                "image": "fluent/fluent-bit",
+                "imagePullPolicy": "Always",
+                "name": "fluent-bit",
+                "resources": {},
+                "stdin": true,
+                "stdinOnce": true,
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "tty": true,
+                "volumeMounts": [
+                    {
+                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
+                        "name": "default-token-9ffht",
+                        "readOnly": true
+                    }
+                ]
+            }
+        ],
+        "dnsPolicy": "ClusterFirst",
+        "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal",
+        "restartPolicy": "Never",
+        "schedulerName": "default-scheduler",
+        "securityContext": {},
+        "serviceAccount": "default",
+        "serviceAccountName": "default",
+        "terminationGracePeriodSeconds": 30,
+        "tolerations": [
+            {
+                "effect": "NoExecute",
+                "key": "node.kubernetes.io/not-ready",
+                "operator": "Exists",
+                "tolerationSeconds": 300
+            },
+            {
+                "effect": "NoExecute",
+                "key": "node.kubernetes.io/unreachable",
+                "operator": "Exists",
+                "tolerationSeconds": 300
+            }
+        ],
+        "volumes": [
+            {
+                "name": "default-token-9ffht",
+                "secret": {
+                    "defaultMode": 420,
+                    "secretName": "default-token-9ffht"
+                }
+            }
+        ]
+    },
+    "status": {
+        "conditions": [
+            {
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-04-03T09:29:00Z",
+                "status": "True",
+                "type": "Initialized"
+            },
+            {
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-04-03T09:29:06Z",
+                "status": "True",
+                "type": "Ready"
+            },
+            {
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-04-03T09:29:00Z",
+                "status": "True",
+                "type": "PodScheduled"
+            }
+        ],
+        "containerStatuses": [
+            {
+                "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16",
+                "image": "fluent/fluent-bit:latest",
+                "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f",
+                "lastState": {},
+                "name": "fluent-bit",
+                "ready": true,
+                "restartCount": 0,
+                "state": {
+                    "running": {
+                        "startedAt": "2019-04-03T09:29:05Z"
+                    }
+                }
+            }
+        ],
+        "hostIP": "10.49.18.80",
+        "phase": "Running",
+        "podIP": "100.116.192.42",
+        "qosClass": "BestEffort",
+        "startTime": "2019-04-03T09:29:00Z"
+    }
+}
diff --git a/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta b/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta
@@ -0,0 +1,109 @@
+{
+	"apiVersion": "v1",
+	"kind": "Pod",
+	"metadata": {},
+	"items": [{
+		"metadata": {
+			"annotations": {
+				"prometheus.io/path": "/api/v1/metrics/prometheus",
+				"prometheus.io/port": "2020",
+				"prometheus.io/scrape": "true"
+			},
+			"creationTimestamp": "2019-04-03T09:29:00Z",
+			"labels": {
+				"app.kubernetes.io/name": "fluent-bit"
+			},
+			"name": "use-kubelet-enabled-pod",
+			"namespace": "options",
+			"resourceVersion": "74466568",
+			"selfLink": "/api/v1/namespaces/core/pods/base",
+			"uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a"
+		},
+		"spec": {
+			"containers": [{
+				"image": "fluent/fluent-bit",
+				"imagePullPolicy": "Always",
+				"name": "fluent-bit",
+				"resources": {},
+				"stdin": true,
+				"stdinOnce": true,
+				"terminationMessagePath": "/dev/termination-log",
+				"terminationMessagePolicy": "File",
+				"tty": true,
+				"volumeMounts": [{
+					"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
+					"name": "default-token-9ffht",
+					"readOnly": true
+				}]
+			}],
+			"dnsPolicy": "ClusterFirst",
+			"nodeName": "ip-10-49-18-80.eu-west-1.compute.internal",
+			"restartPolicy": "Never",
+			"schedulerName": "default-scheduler",
+			"securityContext": {},
+			"serviceAccount": "default",
+			"serviceAccountName": "default",
+			"terminationGracePeriodSeconds": 30,
+			"tolerations": [{
+					"effect": "NoExecute",
+					"key": "node.kubernetes.io/not-ready",
+					"operator": "Exists",
+					"tolerationSeconds": 300
+				},
+				{
+					"effect": "NoExecute",
+					"key": "node.kubernetes.io/unreachable",
+					"operator": "Exists",
+					"tolerationSeconds": 300
+				}
+			],
+			"volumes": [{
+				"name": "default-token-9ffht",
+				"secret": {
+					"defaultMode": 420,
+					"secretName": "default-token-9ffht"
+				}
+			}]
+		},
+		"status": {
+			"conditions": [{
+					"lastProbeTime": null,
+					"lastTransitionTime": "2019-04-03T09:29:00Z",
+					"status": "True",
+					"type": "Initialized"
+				},
+				{
+					"lastProbeTime": null,
+					"lastTransitionTime": "2019-04-03T09:29:06Z",
+					"status": "True",
+					"type": "Ready"
+				},
+				{
+					"lastProbeTime": null,
+					"lastTransitionTime": "2019-04-03T09:29:00Z",
+					"status": "True",
+					"type": "PodScheduled"
+				}
+			],
+			"containerStatuses": [{
+				"containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16",
+				"image": "fluent/fluent-bit:latest",
+				"imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f",
+				"lastState": {},
+				"name": "fluent-bit",
+				"ready": true,
+				"restartCount": 0,
+				"state": {
+					"running": {
+						"startedAt": "2019-04-03T09:29:05Z"
+					}
+				}
+			}],
+			"hostIP": "10.49.18.80",
+			"phase": "Running",
+			"podIP": "100.116.192.42",
+			"qosClass": "BestEffort",
+			"startTime": "2019-04-03T09:29:00Z"
+		}
+	}]
+}
diff --git a/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out
@@ -0,0 +1 @@
+[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-disabled-pod","namespace_name":"options","workload":"use-kubelet-disabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]
diff --git a/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out
@@ -0,0 +1 @@
+[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-enabled-pod","namespace_name":"options","workload":"use-kubelet-enabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]
diff --git a/tests/runtime/filter_kubernetes.c b/tests/runtime/filter_kubernetes.c
@@ -463,6 +463,11 @@ static void flb_test_options_use_kubelet_enabled_daemonset_json()
     flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-daemonset_fluent-bit", NULL, 1);
 }
 
+static void flb_test_options_use_kubelet_enabled_pod_json()
+{
+    flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-pod_fluent-bit", NULL, 1);
+}
+
 static void flb_test_options_use_kubelet_disabled_json()
 {
     flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled_fluent-bit", NULL, 1);
@@ -483,6 +488,11 @@ static void flb_test_options_use_kubelet_disabled_daemonset_json()
     flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-daemonset_fluent-bit", NULL, 1);
 }
 
+static void flb_test_options_use_kubelet_disabled_pod_json()
+{
+    flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-pod_fluent-bit", NULL, 1);
+}
+
 
 #define flb_test_options_merge_log_enabled(target, suffix, nExpected) \
     kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \
@@ -1075,10 +1085,12 @@ TEST_LIST = {
     {"kube_options_use-kubelet_enabled_replicateset_json", flb_test_options_use_kubelet_enabled_replicaset_json},
     {"kube_options_use-kubelet_enabled_deployment_json", flb_test_options_use_kubelet_enabled_deployment_json},
     {"kube_options_use-kubelet_enabled_daemonset_json", flb_test_options_use_kubelet_enabled_daemonset_json},
+    {"kube_options_use-kubelet_enabled_pod_json", flb_test_options_use_kubelet_enabled_pod_json},
     {"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json},
     {"kube_options_use-kubelet_disabled_replicaset_json", flb_test_options_use_kubelet_disabled_replicaset_json},
     {"kube_options_use-kubelet_disabled_deployment_json", flb_test_options_use_kubelet_disabled_deployment_json},
     {"kube_options_use-kubelet_disabled_daemonset_json", flb_test_options_use_kubelet_disabled_daemonset_json},
+{"kube_options_use-kubelet_disabled_pod_json", flb_test_options_use_kubelet_disabled_pod_json},
     {"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text},
     {"kube_options_merge_log_enabled_json", flb_test_options_merge_log_enabled_json},
     {"kube_options_merge_log_enabled_invalid_json", flb_test_options_merge_log_enabled_invalid_json},
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-disabled-pod","namespace_name":"options","workload":"use-kubelet-disabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]