Container id not included in Kubernetes events

[jsoriano]

Generated events coming from kubernetes don't contain the container id, neither on kubernetes.container.id nor on container.id. Tried with autodiscover templates, hints and add_kubernetes_metadata, with Metricbeat and Filebeat, at least since 7.7.0.

Container id is available in autodiscover events as templates containing container.id variables are correctly working.

Reported in a discuss topic, but I could also reproduce it locally on Kind.

For confirmed bugs, please report:

• Version: At least since 7.7.0
• Discuss Forum URL: https://discuss.elastic.co/t/how-to-exclude-other-namespaces/246544/4
• Steps to Reproduce:

Example event, from filebeat 7.9.1 on kind:

{
  "@timestamp": "2020-09-04T10:29:32.956Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.9.1"
  },
  "stream": "stdout",
  "message": "    \"ip\": [",
  "input": {
    "type": "container"
  },
  "kubernetes": {
    "labels": {
      "controller-revision-hash": "744cf49",
      "k8s-app": "filebeat",
      "pod-template-generation": "1"
    },
    "container": {
      "name": "filebeat",
      "image": "docker.elastic.co/beats/filebeat:7.9.1"
    },
    "node": {
      "name": "kind-control-plane"
    },
    "pod": {
      "name": "filebeat-zjrgq",
      "uid": "53819d14-f034-4802-bde7-7eed6e77dec4"
    },
    "namespace": "kube-system"
  },
  "ecs": {
    "version": "1.5.0"
  },
  "host": {
    "hostname": "kind-control-plane",
    "name": "kind-control-plane",
    "architecture": "x86_64",
    "os": {
      "platform": "centos",
      "version": "7 (Core)",
      "family": "redhat",
      "name": "CentOS Linux",
      "kernel": "4.15.0-112-generic",
      "codename": "Core"
    },
    "containerized": true,
    "ip": [
      "10.244.0.1",
      "10.244.0.1",
      "10.244.0.1",
      "192.168.208.2",
      "fc00:f853:ccd:e793::2",
      "fe80::42:c0ff:fea8:d002"
    ],
    "mac": [
      "82:70:43:a4:c2:34",
      "5e:97:4f:33:b7:0f",
      "76:1a:f6:65:54:24",
      "02:42:c0:a8:d0:02"
    ]
  },
  "agent": {
    "name": "kind-control-plane",
    "type": "filebeat",
    "version": "7.9.1",
    "hostname": "kind-control-plane",
    "ephemeral_id": "a7f64ae4-ab9c-4906-b2c7-7ed43db31f71",
    "id": "cd60a75f-2020-4ccf-a6de-63cbf68d57b7"
  },
  "log": {
    "offset": 23561,
    "file": {
      "path": "/var/log/containers/filebeat-zjrgq_kube-system_filebeat-e28d00c32477c1ff3566242b0f1b4e6bcabfa6e066edea44718e2484264852b0.log"
    }
  }
}

From Metricbeat:

{
  "@timestamp": "2020-09-04T10:40:28.053Z",
  "@metadata": {
    "beat": "metricbeat",
    "type": "_doc",
    "version": "7.9.1"
  },
  "agent": {
    "ephemeral_id": "21876d5e-1649-4259-8e06-d278f940a685",
    "id": "b2a6815c-0653-40a9-b237-1519db63dc08",
    "name": "kind-control-plane",
    "type": "metricbeat",
    "version": "7.9.1",
    "hostname": "kind-control-plane"
  },
  "metricset": {
    "name": "info",
    "period": 60000
  },
  "event": {
    "duration": 1009782,
    "dataset": "redis.info",
    "module": "redis"
  },
  "service": {
    "version": "6.0.7",
    "address": "10.244.0.9:6379",
    "type": "redis"
  },
  "kubernetes": {
    "container": {
      "name": "redis",
      "image": "redis"
    },
    "node": {
      "name": "kind-control-plane"
    },
    "pod": {
      "name": "redis-5656ffd667-p45xh",
      "uid": "77239772-4f1f-41ff-9a35-86b84c78bbf6"
    },
    "namespace": "default",
    "replicaset": {
      "name": "redis-5656ffd667"
    },
    "labels": {
      "pod-template-hash": "5656ffd667",
      "run": "redis"
    }
  },
  "host": {
    "name": "kind-control-plane"
  },
  "redis": {
....
  },
  "process": {
    "pid": 1
  },
  "os": {
    "full": "Linux 4.15.0-112-generic x86_64"
  },
  "ecs": {
    "version": "1.5.0"
  }
}

[elasticmachine]

Pinging @elastic/integrations-platforms (Team:Platforms)