diff --git a/plugins/filter_kubernetes/kube_meta.c b/plugins/filter_kubernetes/kube_meta.c index 559ef9003a7..afa897a0ba0 100644 --- a/plugins/filter_kubernetes/kube_meta.c +++ b/plugins/filter_kubernetes/kube_meta.c @@ -737,12 +737,24 @@ static void cb_results_workload(const char *name, const char *value, } } +/* + * Search workload based on the following priority + * where the top is highest priority + * 1. Deployment name + * 2. StatefulSet name + * 3. DaemonSet name + * 4. Job name + * 5. CronJob name + * 6. Pod name + * 7. Container name + */ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgpack_object map) { int i,j,ownerIndex; int regex_found; int replicaset_match; int podname_match = FLB_FALSE; + int workload_found = FLB_FALSE; msgpack_object k, v; msgpack_object_map ownerMap; struct flb_regex_search result; @@ -788,6 +800,7 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp if (strncmp(key.via.str.ptr, "name", key.via.str.size) == 0) { /* Store the value of 'name' in workload_val so it can be reused by set_workload */ workload_val = val; + workload_found = FLB_TRUE; if (replicaset_match) { regex_found = flb_regex_do(ctx->deploymentRegex, val.via.str.ptr, val.via.str.size, &result); if (regex_found > 0) { @@ -808,6 +821,17 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp } } } + if(!workload_found) { + if(meta->podname != NULL) { + meta->workload = flb_strndup(meta->podname, meta->podname_len); + meta->workload_len = meta->podname_len; + meta->fields++; + } else if (meta->container_name != NULL) { + meta->workload = flb_strndup(meta->container_name, meta->container_name_len); + meta->workload_len = meta->container_name_len; + meta->fields++; + } + } return; diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index 868b71e5385..ff3f8243e70 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -969,6 +969,10 @@ void parse_entity(struct flb_cloudwatch *ctx, entity *entity, msgpack_object map } } } + if(entity->key_attributes->name == NULL && entity->attributes->name_source == NULL &&entity->attributes->workload != NULL) { + entity->key_attributes->name = flb_strndup(entity->attributes->workload, strlen(entity->attributes->workload)); + entity->attributes->name_source = flb_strndup("K8sWorkload", 11); + } } void update_or_create_entity(struct flb_cloudwatch *ctx, struct log_stream *stream, const msgpack_object map) { diff --git a/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log new file mode 100644 index 00000000000..259723131be --- /dev/null +++ b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log @@ -0,0 +1 @@ +{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log new file mode 100644 index 00000000000..259723131be --- /dev/null +++ b/tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log @@ -0,0 +1 @@ +{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"} diff --git a/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta b/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta new file mode 100644 index 00000000000..a8bcc69017f --- /dev/null +++ b/tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta @@ -0,0 +1,116 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "prometheus.io/path": "/api/v1/metrics/prometheus", + "prometheus.io/port": "2020", + "prometheus.io/scrape": "true" + }, + "creationTimestamp": "2019-04-03T09:29:00Z", + "labels": { + "app.kubernetes.io/name": "fluent-bit" + }, + "name": "use-kubelet-disabled-pod", + "namespace": "options", + "resourceVersion": "74466568", + "selfLink": "/api/v1/namespaces/core/pods/base", + "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a" + }, + "spec": { + "containers": [ + { + "image": "fluent/fluent-bit", + "imagePullPolicy": "Always", + "name": "fluent-bit", + "resources": {}, + "stdin": true, + "stdinOnce": true, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-9ffht", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal", + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "default-token-9ffht", + "secret": { + "defaultMode": 420, + "secretName": "default-token-9ffht" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:06Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16", + "image": "fluent/fluent-bit:latest", + "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f", + "lastState": {}, + "name": "fluent-bit", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2019-04-03T09:29:05Z" + } + } + } + ], + "hostIP": "10.49.18.80", + "phase": "Running", + "podIP": "100.116.192.42", + "qosClass": "BestEffort", + "startTime": "2019-04-03T09:29:00Z" + } +} diff --git a/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta b/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta new file mode 100644 index 00000000000..122479e9c07 --- /dev/null +++ b/tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta @@ -0,0 +1,109 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {}, + "items": [{ + "metadata": { + "annotations": { + "prometheus.io/path": "/api/v1/metrics/prometheus", + "prometheus.io/port": "2020", + "prometheus.io/scrape": "true" + }, + "creationTimestamp": "2019-04-03T09:29:00Z", + "labels": { + "app.kubernetes.io/name": "fluent-bit" + }, + "name": "use-kubelet-enabled-pod", + "namespace": "options", + "resourceVersion": "74466568", + "selfLink": "/api/v1/namespaces/core/pods/base", + "uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a" + }, + "spec": { + "containers": [{ + "image": "fluent/fluent-bit", + "imagePullPolicy": "Always", + "name": "fluent-bit", + "resources": {}, + "stdin": true, + "stdinOnce": true, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "volumeMounts": [{ + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-9ffht", + "readOnly": true + }] + }], + "dnsPolicy": "ClusterFirst", + "nodeName": "ip-10-49-18-80.eu-west-1.compute.internal", + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [{ + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [{ + "name": "default-token-9ffht", + "secret": { + "defaultMode": 420, + "secretName": "default-token-9ffht" + } + }] + }, + "status": { + "conditions": [{ + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:06Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2019-04-03T09:29:00Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [{ + "containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16", + "image": "fluent/fluent-bit:latest", + "imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f", + "lastState": {}, + "name": "fluent-bit", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2019-04-03T09:29:05Z" + } + } + }], + "hostIP": "10.49.18.80", + "phase": "Running", + "podIP": "100.116.192.42", + "qosClass": "BestEffort", + "startTime": "2019-04-03T09:29:00Z" + } + }] +} diff --git a/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out new file mode 100644 index 00000000000..8aea6e523eb --- /dev/null +++ b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-disabled-pod","namespace_name":"options","workload":"use-kubelet-disabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}] diff --git a/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out new file mode 100644 index 00000000000..ccc0d77b4d0 --- /dev/null +++ b/tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out @@ -0,0 +1 @@ +[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-enabled-pod","namespace_name":"options","workload":"use-kubelet-enabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}] diff --git a/tests/runtime/filter_kubernetes.c b/tests/runtime/filter_kubernetes.c index 9ce4b6bc947..99476cf7dd5 100644 --- a/tests/runtime/filter_kubernetes.c +++ b/tests/runtime/filter_kubernetes.c @@ -463,6 +463,11 @@ static void flb_test_options_use_kubelet_enabled_daemonset_json() flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-daemonset_fluent-bit", NULL, 1); } +static void flb_test_options_use_kubelet_enabled_pod_json() +{ + flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-pod_fluent-bit", NULL, 1); +} + static void flb_test_options_use_kubelet_disabled_json() { flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled_fluent-bit", NULL, 1); @@ -483,6 +488,11 @@ static void flb_test_options_use_kubelet_disabled_daemonset_json() flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-daemonset_fluent-bit", NULL, 1); } +static void flb_test_options_use_kubelet_disabled_pod_json() +{ + flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-pod_fluent-bit", NULL, 1); +} + #define flb_test_options_merge_log_enabled(target, suffix, nExpected) \ kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \ @@ -1075,10 +1085,12 @@ TEST_LIST = { {"kube_options_use-kubelet_enabled_replicateset_json", flb_test_options_use_kubelet_enabled_replicaset_json}, {"kube_options_use-kubelet_enabled_deployment_json", flb_test_options_use_kubelet_enabled_deployment_json}, {"kube_options_use-kubelet_enabled_daemonset_json", flb_test_options_use_kubelet_enabled_daemonset_json}, + {"kube_options_use-kubelet_enabled_pod_json", flb_test_options_use_kubelet_enabled_pod_json}, {"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json}, {"kube_options_use-kubelet_disabled_replicaset_json", flb_test_options_use_kubelet_disabled_replicaset_json}, {"kube_options_use-kubelet_disabled_deployment_json", flb_test_options_use_kubelet_disabled_deployment_json}, {"kube_options_use-kubelet_disabled_daemonset_json", flb_test_options_use_kubelet_disabled_daemonset_json}, +{"kube_options_use-kubelet_disabled_pod_json", flb_test_options_use_kubelet_disabled_pod_json}, {"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text}, {"kube_options_merge_log_enabled_json", flb_test_options_merge_log_enabled_json}, {"kube_options_merge_log_enabled_invalid_json", flb_test_options_merge_log_enabled_invalid_json},