diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c53e6a7ac9d..caa9f5188c5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -860,6 +860,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add io.ops in fields exported by system.diskio. {pull}22066[22066] - `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732] - Adjust the Apache status fields in the fleet mode. {pull}22821[22821] +- Add AWS Fargate overview dashboard. {pull}22941[22941] - Add process.state, process.cpu.pct, process.cpu.start_time and process.memory.pct. {pull}22845[22845] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 67ec55b7fa9..585689db530 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4604,6 +4604,16 @@ type: double +*`task_stats.identifier`*:: ++ +-- +Container identifier across tasks and clusters, which equals to container.name + '/' + container.id. + + +type: keyword + +-- + [float] === cpu diff --git a/metricbeat/docs/images/metricbeat-awsfargate-overview.png b/metricbeat/docs/images/metricbeat-awsfargate-overview.png new file mode 100644 index 00000000000..863d486c14b Binary files /dev/null and b/metricbeat/docs/images/metricbeat-awsfargate-overview.png differ diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 847a13703ce..3eee714ce1d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -32,7 +32,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .11+| .11+| |<> beta[] diff --git a/x-pack/metricbeat/module/awsfargate/_meta/kibana/7/dashboard/Metricbeat-awsfargate-overview.json b/x-pack/metricbeat/module/awsfargate/_meta/kibana/7/dashboard/Metricbeat-awsfargate-overview.json new file mode 100644 index 00000000000..05feb208f00 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/kibana/7/dashboard/Metricbeat-awsfargate-overview.json @@ -0,0 +1,1235 @@ +{ + "objects": [ + { + "attributes": { + "description": "Overview of AWS Fargate Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "ECS Task Filter" + }, + "gridData": { + "h": 14, + "i": "2429ae11-ccb3-4016-b18d-c88140e29f67", + "w": 9, + "x": 0, + "y": 0 + }, + "panelIndex": "2429ae11-ccb3-4016-b18d-c88140e29f67", + "panelRefName": "panel_0", + "title": "ECS Task Filter", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Containers" + }, + "gridData": { + "h": 14, + "i": "735a25cb-f781-4a8b-99e6-2adf582f8a37", + "w": 39, + "x": 9, + "y": 0 + }, + "panelIndex": "735a25cb-f781-4a8b-99e6-2adf582f8a37", + "panelRefName": "panel_1", + "title": "Containers", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Total Number of Containers" + }, + "gridData": { + "h": 15, + "i": "5d4537e8-9aee-47e6-8580-1961d8a3a38e", + "w": 24, + "x": 0, + "y": 14 + }, + "panelIndex": "5d4537e8-9aee-47e6-8580-1961d8a3a38e", + "panelRefName": "panel_2", + "title": "Total Number of Containers", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "CPU Utilization Per Cluster" + }, + "gridData": { + "h": 15, + "i": "d3c9208e-c699-4514-a091-a731d28ab9af", + "w": 24, + "x": 24, + "y": 14 + }, + "panelIndex": "d3c9208e-c699-4514-a091-a731d28ab9af", + "panelRefName": "panel_3", + "title": "CPU Utilization Per Cluster", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Container Memory Usage Pct" + }, + "gridData": { + "h": 15, + "i": "9dd2b002-5170-49ea-afc2-30b0b46b0fd7", + "w": 24, + "x": 0, + "y": 29 + }, + "panelIndex": "9dd2b002-5170-49ea-afc2-30b0b46b0fd7", + "panelRefName": "panel_4", + "title": "Container Memory Usage Pct", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "DiskIO Write" + }, + "gridData": { + "h": 15, + "i": "89b0cd8b-13d9-4a43-8f93-3410aff125c3", + "w": 24, + "x": 24, + "y": 44 + }, + "panelIndex": "89b0cd8b-13d9-4a43-8f93-3410aff125c3", + "panelRefName": "panel_5", + "title": "DiskIO Write", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "DiskIO Read" + }, + "gridData": { + "h": 15, + "i": "d7a6623a-14cf-411c-ab73-ded3734a359b", + "w": 24, + "x": 0, + "y": 44 + }, + "panelIndex": "d7a6623a-14cf-411c-ab73-ded3734a359b", + "panelRefName": "panel_6", + "title": "DiskIO Read", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Container CPU Total Norm Pct" + }, + "gridData": { + "h": 15, + "i": "a7a6e425-98fd-4210-b3d5-005268be6274", + "w": 24, + "x": 24, + "y": 29 + }, + "panelIndex": "a7a6e425-98fd-4210-b3d5-005268be6274", + "panelRefName": "panel_7", + "title": "Container CPU Total Norm Pct", + "version": "7.10.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat AWSFargate] Fargate Overview", + "version": 1 + }, + "id": "20dc7c50-2e89-11eb-991c-c5fd3b7f5a63", + "migrationVersion": { + "dashboard": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "38eee240-2e90-11eb-a417-f961313c14f6", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "8cf592f0-2e8e-11eb-a417-f961313c14f6", + "name": "panel_1", + "type": "lens" + }, + { + "id": "86720980-3966-11eb-a417-f961313c14f6", + "name": "panel_2", + "type": "lens" + }, + { + "id": "b1837e70-396f-11eb-a417-f961313c14f6", + "name": "panel_3", + "type": "lens" + }, + { + "id": "d3f3aa80-2e8d-11eb-a417-f961313c14f6", + "name": "panel_4", + "type": "lens" + }, + { + "id": "c3952840-2e91-11eb-a417-f961313c14f6", + "name": "panel_5", + "type": "lens" + }, + { + "id": "a7970500-2e91-11eb-a417-f961313c14f6", + "name": "panel_6", + "type": "lens" + }, + { + "id": "60941700-2e88-11eb-991c-c5fd3b7f5a63", + "name": "panel_7", + "type": "lens" + } + ], + "type": "dashboard", + "updated_at": "2020-12-08T16:33:09.074Z", + "version": "WzEzNzIwNywxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "ECS Task Filter [Metricbeat AWSFargate]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "cloud.region", + "id": "1607442436907", + "indexPatternRefName": "control_0_index_pattern", + "label": "Region Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "awsfargate.task_stats.cluster_name", + "id": "1607442399749", + "indexPatternRefName": "control_1_index_pattern", + "label": "Cluster Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "container.labels.com_amazonaws_ecs_task-definition-family", + "id": "1606248025028", + "indexPatternRefName": "control_2_index_pattern", + "label": "Task Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": false, + "useTimeFilter": false + }, + "title": "ECS Task Filter [Metricbeat AWSFargate]", + "type": "input_control_vis" + } + }, + "id": "38eee240-2e90-11eb-a417-f961313c14f6", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_2_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-12-08T15:47:56.120Z", + "version": "WzEzNjYzOCwxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e350c9e6-e6b7-40f1-83ed-94a38ed9467a": { + "columnOrder": [ + "040803fd-23f9-4f4f-95da-e8be99becbc1", + "eb452b61-bf12-46c0-afae-c0f1c776156c", + "b1e08425-6a1e-4db6-bb47-a65167b7e14f", + "09de0d9e-c174-46cf-bdc8-50158475b4a7", + "e0782143-0636-43b4-8fac-b857ea5282b2", + "7626706b-6efa-4fe7-b45c-6e9afa07f982", + "441ceb32-53e7-4408-9638-6dfd38202144" + ], + "columns": { + "040803fd-23f9-4f4f-95da-e8be99becbc1": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Task Name", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "container.labels.com_amazonaws_ecs_task-definition-family" + }, + "09de0d9e-c174-46cf-bdc8-50158475b4a7": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "CPU", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 3 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.cpu.total.norm.pct" + }, + "441ceb32-53e7-4408-9638-6dfd38202144": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Memory", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 3 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.memory.usage.pct" + }, + "7626706b-6efa-4fe7-b45c-6e9afa07f982": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Memory Usage", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.memory.usage.total" + }, + "b1e08425-6a1e-4db6-bb47-a65167b7e14f": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "ECS Cluster ARN", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "container.labels.com_amazonaws_ecs_cluster" + }, + "e0782143-0636-43b4-8fac-b857ea5282b2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "DiskIO", + "operationType": "avg", + "scale": "ratio", + "sourceField": "awsfargate.task_stats.diskio.total" + }, + "eb452b61-bf12-46c0-afae-c0f1c776156c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Container Name", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "container.name" + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "columns": [ + "eb452b61-bf12-46c0-afae-c0f1c776156c", + "040803fd-23f9-4f4f-95da-e8be99becbc1", + "b1e08425-6a1e-4db6-bb47-a65167b7e14f", + "09de0d9e-c174-46cf-bdc8-50158475b4a7", + "e0782143-0636-43b4-8fac-b857ea5282b2", + "7626706b-6efa-4fe7-b45c-6e9afa07f982", + "441ceb32-53e7-4408-9638-6dfd38202144" + ], + "layerId": "e350c9e6-e6b7-40f1-83ed-94a38ed9467a" + } + ] + } + }, + "title": "Container Table [Metricbeat AWSFargate]", + "visualizationType": "lnsDatatable" + }, + "id": "8cf592f0-2e8e-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-e350c9e6-e6b7-40f1-83ed-94a38ed9467a", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-04T21:19:50.812Z", + "version": "WzM4ODc5LDFd" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "450490b7-c7f9-42bf-8e7d-227cbf960e94": { + "columnOrder": [ + "b20aebaf-7126-4b1c-acca-79cfa113b44d", + "915eda3d-00ce-40d0-ada8-4eb372202b42" + ], + "columns": { + "915eda3d-00ce-40d0-ada8-4eb372202b42": { + "dataType": "number", + "isBucketed": false, + "label": "Unique count of container.id", + "operationType": "cardinality", + "scale": "ratio", + "sourceField": "container.id" + }, + "b20aebaf-7126-4b1c-acca-79cfa113b44d": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "915eda3d-00ce-40d0-ada8-4eb372202b42" + ], + "layerId": "450490b7-c7f9-42bf-8e7d-227cbf960e94", + "position": "top", + "seriesType": "bar_stacked", + "showGridlines": false, + "xAccessor": "b20aebaf-7126-4b1c-acca-79cfa113b44d" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + } + } + }, + "title": "Total Number of Containers [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "86720980-3966-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-450490b7-c7f9-42bf-8e7d-227cbf960e94", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T15:03:26.999Z", + "version": "WzEzNjU3MywxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "82f43ff6-6ac1-4f73-b532-61b6b9fe15b0": { + "columnOrder": [ + "e139ccbc-54ff-4921-b345-e3c3ff6a9efc", + "d7b9836e-d606-4aab-b2d8-44cbc726a950" + ], + "columns": { + "d7b9836e-d606-4aab-b2d8-44cbc726a950": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "CPU Utilization", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.cpu.total.norm.pct", + "suggestedPriority": 0 + }, + "e139ccbc-54ff-4921-b345-e3c3ff6a9efc": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Cluster Name", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "d7b9836e-d606-4aab-b2d8-44cbc726a950", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "awsfargate.task_stats.cluster_name" + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "d7b9836e-d606-4aab-b2d8-44cbc726a950" + ], + "layerId": "82f43ff6-6ac1-4f73-b532-61b6b9fe15b0", + "seriesType": "bar_horizontal", + "xAccessor": "e139ccbc-54ff-4921-b345-e3c3ff6a9efc" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_horizontal", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + } + } + }, + "title": "CPU Utilization Per Cluster [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "b1837e70-396f-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-82f43ff6-6ac1-4f73-b532-61b6b9fe15b0", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T16:09:04.727Z", + "version": "WzEzNjkxNiwxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e350c9e6-e6b7-40f1-83ed-94a38ed9467a": { + "columnOrder": [ + "55121be2-80f6-407c-9352-3e13705ac860", + "eb452b61-bf12-46c0-afae-c0f1c776156c", + "5b846eff-5ca7-45b0-b4c2-0346e9986302" + ], + "columns": { + "55121be2-80f6-407c-9352-3e13705ac860": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "10s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "5b846eff-5ca7-45b0-b4c2-0346e9986302": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Container Memory Usage Pct", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.memory.usage.pct", + "suggestedPriority": 0 + }, + "eb452b61-bf12-46c0-afae-c0f1c776156c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "container identifier", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "5b846eff-5ca7-45b0-b4c2-0346e9986302", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "awsfargate.task_stats.identifier" + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "5b846eff-5ca7-45b0-b4c2-0346e9986302" + ], + "layerId": "e350c9e6-e6b7-40f1-83ed-94a38ed9467a", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "eb452b61-bf12-46c0-afae-c0f1c776156c", + "xAccessor": "55121be2-80f6-407c-9352-3e13705ac860" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + } + } + }, + "title": "Container Memory Usage Pct [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "d3f3aa80-2e8d-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-e350c9e6-e6b7-40f1-83ed-94a38ed9467a", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T16:19:29.265Z", + "version": "WzEzNzExMSwxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c8d70f88-6554-41ca-ac1e-a3cf8b992972": { + "columnOrder": [ + "0505d890-632e-4434-a68f-f019e7893eaf", + "989ce63a-7fcd-4c83-980b-d595a1dd2469", + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b" + ], + "columns": { + "0505d890-632e-4434-a68f-f019e7893eaf": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of awsfargate.task_stats.identifier", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "awsfargate.task_stats.identifier" + }, + "989ce63a-7fcd-4c83-980b-d595a1dd2469": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "DiskIO Write", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.diskio.write.bytes", + "suggestedPriority": 0 + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b" + ], + "layerId": "c8d70f88-6554-41ca-ac1e-a3cf8b992972", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "0505d890-632e-4434-a68f-f019e7893eaf", + "xAccessor": "989ce63a-7fcd-4c83-980b-d595a1dd2469" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + } + } + }, + "title": "DiskIO Write [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "c3952840-2e91-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-c8d70f88-6554-41ca-ac1e-a3cf8b992972", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T16:13:22.119Z", + "version": "WzEzNzAxMSwxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c8d70f88-6554-41ca-ac1e-a3cf8b992972": { + "columnOrder": [ + "0505d890-632e-4434-a68f-f019e7893eaf", + "989ce63a-7fcd-4c83-980b-d595a1dd2469", + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b" + ], + "columns": { + "0505d890-632e-4434-a68f-f019e7893eaf": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of awsfargate.task_stats.identifier", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "awsfargate.task_stats.identifier" + }, + "989ce63a-7fcd-4c83-980b-d595a1dd2469": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "DiskIO Read", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.diskio.read.bytes", + "suggestedPriority": 0 + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "ab00dce2-6c01-4a0d-8d7c-71330afbaa3b" + ], + "layerId": "c8d70f88-6554-41ca-ac1e-a3cf8b992972", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "0505d890-632e-4434-a68f-f019e7893eaf", + "xAccessor": "989ce63a-7fcd-4c83-980b-d595a1dd2469" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + } + } + }, + "title": "DiskIO Read [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "a7970500-2e91-11eb-a417-f961313c14f6", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-c8d70f88-6554-41ca-ac1e-a3cf8b992972", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T16:13:21.224Z", + "version": "WzEzNjk5NSwxXQ==" + }, + { + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "0749cbce-4a88-471d-9715-9f4f54510864": { + "columnOrder": [ + "35481ca7-cebb-495c-974f-34ede5f3935c", + "33d4857c-800a-4854-9369-dbf019501bdd", + "ec425562-fe79-4326-b753-20b408c806fb" + ], + "columns": { + "33d4857c-800a-4854-9369-dbf019501bdd": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Container Identifier", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "ec425562-fe79-4326-b753-20b408c806fb", + "type": "column" + }, + "orderDirection": "desc", + "size": 20 + }, + "scale": "ordinal", + "sourceField": "awsfargate.task_stats.identifier" + }, + "35481ca7-cebb-495c-974f-34ede5f3935c": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "10s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "ec425562-fe79-4326-b753-20b408c806fb": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Container CPU Total Norm Pct", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "awsfargate.task_stats.cpu.total.norm.pct", + "suggestedPriority": 0 + } + } + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "ec425562-fe79-4326-b753-20b408c806fb" + ], + "layerId": "0749cbce-4a88-471d-9715-9f4f54510864", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "33d4857c-800a-4854-9369-dbf019501bdd", + "xAccessor": "35481ca7-cebb-495c-974f-34ede5f3935c" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line" + } + }, + "title": "Container CPU Total Norm Pct [Metricbeat AWSFargate]", + "visualizationType": "lnsXY" + }, + "id": "60941700-2e88-11eb-991c-c5fd3b7f5a63", + "migrationVersion": { + "lens": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-0749cbce-4a88-471d-9715-9f4f54510864", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2020-12-08T16:30:52.054Z", + "version": "WzEzNzE5OSwxXQ==" + } + ], + "version": "7.10.0" +} diff --git a/x-pack/metricbeat/module/awsfargate/awsfargate.go b/x-pack/metricbeat/module/awsfargate/awsfargate.go index 94a030b90ca..7fae572f81a 100644 --- a/x-pack/metricbeat/module/awsfargate/awsfargate.go +++ b/x-pack/metricbeat/module/awsfargate/awsfargate.go @@ -38,7 +38,7 @@ func newModule(base mb.BaseModule) (mb.Module, error) { return &base, nil } -// NewMetricSet creates a base metricset for awsfargate metricsets +// NewMetricSet creates a base metricset for awsfargate metricset func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { var config Config err := base.Module().UnpackConfig(&config) diff --git a/x-pack/metricbeat/module/awsfargate/cloudformation.yml b/x-pack/metricbeat/module/awsfargate/cloudformation.yml index 18e0759414a..ac43e044edc 100644 --- a/x-pack/metricbeat/module/awsfargate/cloudformation.yml +++ b/x-pack/metricbeat/module/awsfargate/cloudformation.yml @@ -2,22 +2,36 @@ AWSTemplateFormatVersion: "2010-09-09" Parameters: SubnetID: Type: String + CloudIDArn: + Type: String + CloudAuthArn: + Type: String + ClusterName: + Type: String + RoleName: + Type: String + TaskName: + Type: String + ServiceName: + Type: String + LogGroupName: + Type: String Resources: Cluster: Type: AWS::ECS::Cluster Properties: - ClusterName: metricbeat-cloudformation-fargate + ClusterName: !Ref ClusterName ClusterSettings: - Name: containerInsights Value: enabled LogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: metricbeat-fargate-log-group + LogGroupName: !Ref LogGroupName ExecutionRole: Type: AWS::IAM::Role Properties: - RoleName: ecsFargateTaskExecutionRole + RoleName: !Ref RoleName AssumeRolePolicyDocument: Statement: - Effect: Allow @@ -35,24 +49,24 @@ Resources: Action: - secretsmanager:GetSecretValue Resource: - - - - + - !Ref CloudIDArn + - !Ref CloudAuthArn TaskDefinition: Type: AWS::ECS::TaskDefinition Properties: - Family: deployment-task-metricbeat + Family: !Ref TaskName Cpu: 256 Memory: 512 NetworkMode: awsvpc ExecutionRoleArn: !Ref ExecutionRole ContainerDefinitions: - - Name: deployment-task-metricbeat-container - Image: kaiyansheng/metricbeat-awsfargate:v1 + - Name: metricbeat-container + Image: docker.elastic.co/beats/metricbeat:7.11.0-SNAPSHOT Secrets: - Name: ELASTIC_CLOUD_ID - ValueFrom: + ValueFrom: !Ref CloudIDArn - Name: ELASTIC_CLOUD_AUTH - ValueFrom: + ValueFrom: !Ref CloudAuthArn LogConfiguration: LogDriver: awslogs Options: @@ -63,14 +77,25 @@ Resources: - sh - -c Command: - - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + - ./metricbeat setup -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + - Name: stress-test + Image: containerstack/alpine-stress + Essential: false + DependsOn: + - ContainerName: metricbeat-container + Condition: START + EntryPoint: + - sh + - -c + Command: + - stress --cpu 8 --io 4 --vm 2 --vm-bytes 128M --timeout 6000s RequiresCompatibilities: - EC2 - FARGATE Service: Type: AWS::ECS::Service Properties: - ServiceName: deployment-metricbeat-service + ServiceName: !Ref ServiceName Cluster: !Ref Cluster TaskDefinition: !Ref TaskDefinition DesiredCount: 1 diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index 9659165097f..66eb2c42d53 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded gzipped contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWk1z2zYQvftX7Hh6yjTKJdODDp1RnWTGB9sZfzRHGQJXMioSYAAwitLpf+8ABCRKAimQsmTWDY8iwH1v9+0uPvQW5rgcAlmoKZEzovEMQDOd4hDOR1/u4FP56/kZQIKKSpZrJvgQfj8DAHhcz3uETCRFikBFmiLVCsx09xIy1JJRBVMpMtBEzc0vJCGaAPIkF4zrwRmAxBSJwiFMUJMzgCnDNFFDa8s8b4GTDIf2C2OliVarVwB6meMQZlIUeeXXAGr/PK4/8whUcE0YV6Cf1nD1E9GwQImgqCQ5JiWBKjNLxn4Dfvn748Xd+OLm+n50ef3xdnz18X70YXQ/Gj/cXo7/fP/POzP2XTk2SN8/227wz7Y7qi6hebHxe50/dnxyW3DNMoSLzw+e92BrfMhu1fYcJcd0kFO9M8TDUJSkmIynqSChQVMhM6KHkKOkyEMjGuLon8/lZDJDEFOwpBh34EDlhOI2sR0KXMjsP8ADDE6Ssh+YwGRpJcuLbILSTDCBpELiThh32GpG56qWair4rBsBg8B+O97/aqk09s71QpPUsSmDYDztoO6j0kcpNfA5VFKO9TEl5YBaE/VACoWyb3533jbQ9iWChd9H7exwOFQxlulJSlCM321uvLjP71cZWigy2wu3FzrZwtxFFp5VwtSciW7rmA9MzeHy3U3XRYxEktT6MWQ90j0jSousSInGxNpQkBSS8Zn1TMqmZXY9oV+BogwFqgZ8lYDIQ4kE+5IpkoZ5rldx7ExkDXey1NgZsBdu00ciSf1hPmEJHcZHlpunTnRa+58WUiLXLg65KW9IBQ8JuNKiUX5jFMemlB8ZaVkTbM/QwhuGyxuQ+LVApdWvpi5zwkWJOxxDD3xBmD4xao8TVG78bBAYaTAOXwssUBkFel6tuNjp4Tg9N5F12S2Nrkg169hIqhHfnqZS4swlUlPzhvDb4P2Lyn4h2Z7MrKvuLTBWq7w12LE6Npd6iCr3EKOkFtxgIwaHsYur/9EMYvpAS6ZlPzAsNfLDaTa0hWiWneLkc8XFa0+yQLs+cQzoz9MzoEXfODKLZ+8hENtHjkesW08pNdjzphKXKKskKbKMyOUp+0rZ9Qh/7R3GbOJEjpKYaa++09idhw/q62g5leD9bDsnZ/Gz7WwckvW06+wU85pM8VwyzMROu4k8Gruyc7sejNkLy8Gb2rMxMfkLg0eQ5YtxoyIqY8YZyXPGZ27C+Zvzbgdut2ThvOXuZctLZ1NCraeUezswb4FxjXJKaChZY07dqMgyVqedZ1sGXFgr2h6pGrEIDl8YT8SiLmNje3xTisAL9sYy7Zs/6DnkSOY9pPAZydzJIzpwK0qSfSMaxwsh5yYlFOrB/mp2mgNThw0cNlCoo3lNCUsHVBTBuwZoV5ajwH4iLAVrEOXubUoVWsr25fFp3OuKtYXTjFiq5i3VM9Sd27u7jVoaBvRayo1jKlGxxCydFGpQ7EfgGm6bV/gObpNVhKgh6k6uA7+rGmbeSvCucZukvd7bS7NJdS1RP9j7xBj9QYQGq1Qy8r1xXKQSoYUaoR1381yR7559zXVw9YnTIrTVI7TSJHRgWWEYpUdoUVCgJ6HcKDCBYHo+HLXpq93W+dfl5K4L/aalcIlgjsuFkKHtX4Q7PLqVGWu2/i8OjE9EETw3OPhO/pJTkZm1i3O32ygoxiluHraYN1JjElJjzPagL7fc29to5j1gv928zkikyPOT31k6q2ukOaFzrGsBHitKKeSx/wdRQi1NmZVvO4hu0InduR/j6ti30EdLvJtCz8T/OvGE90DvE2+FtL+J1w7iyyTeLsZ/AwAA//8SjLaw" + return "eJzsWk1v2zgTvudXDIIXKNAP51K8Bx8W8KYtkEOSIh/bo0NTY4driVRIqq672P++ICXask1JlBw72mx5NL+eZ+aZGYr0B5jjcghkoaZEzojGEwDNdIxDOB19u4Uv+a+nJwARKipZqpngQ/jtBADgYT3vARIRZTECFXGMVCsw04tOSFBLRhVMpUhAEzU3v5CIaALIo1QwrgcnABJjJAqHMEFNTgCmDONIDe1epn0AThIc2hXGShOtVl0AepniEGZSZGnpVw9q1x7WyzwAFVwTxhXoxzVc/Ug0LFAiKCpJilFOoMzMkrFrwP/++nx+Oz6/vrobXVx9vhlffr4bfRrdjcb3NxfjPz7+fWbGnuVjvfRd2zaDa9vmKJuERcg1mzKUG93OLHNcLoSMtvpqjGPaeW4TlKXVgVAplLK8FRAeAY0zpVGq97B4ZPQR8CkjsQItnE1RDgxGeAdvzt7Au9LPLBp4ydA087LYdu4Oh5uMa5YgnH+9d04cbI33GbG89xwlx3iQUr0zxMFQlMQYjaexIL5BUyETooeQoqTIfSMa7G7a13wymSGIKVhSjBfgQKWE4jaxHQpcyORfwAMMThKznxjBZGnjj2fJBKWZYBxJhcQdN+6w1YzOVSXVWPBZNwIGgV073P5qqTT2zvRCk7hgkzvBWLqA2kSlj1Kq4bOvpArWh5RUAdRuUQ0kUyj7ZvfC2gZaUyBY+H3Uzg6HfRVjmR4lBYXY3cbGi9v8bhWhmSKzRri90MkW5i6ycKwipuZMdDvHfGJqDhdn110PMRLJ9lmvfvdA84wozZIsJhoju4eCKJOMz6xlYjbNo+sR12c8n6MqwJcJiNQXSNAUTIE0TLta+bEzkTXcyVJjZ8BOuHWLBJL63SxhCe3HR+Zfgp3otLY/zaRErgs/pCa9IRXcJ+BSiUb5nVEcm1R+YKR5TrA1Qwu3MVxcg8SnDJVW701e5oSLHLffhw74gjB9ZNQOJ6jU2NkgMNJgHJ4yzFAZBTperbjY6X4/PTeRddrNN12RqtexkVQtvoaikuNMJVKT84bw/8HHF5X9QrKGyKzK7i0wlrO83bBjdqxP9RCU7iFESS24wYYP9mMXlv+DGYTUgZZM83pgWGrk+9OsKQvBLDv5ycVK4a+GYIF2deIQ0J+nZkCLunFgFs9eQyC0jhyOWLeakmuw50UlLFBWQZIlCZHLY9aVvOoR/torjPmIEylKYqa9+kpjvzycU19HySk571fZOTqLX2Vn45Ksp1VnJ5lXRIrjkmAidspN4NXYpZ3b9WLMvr4O3lbejYnJn+i9gsw7xrWKKI0ZJyRNGZ8VE07fnna7cLshi8JaxSNz/oJuUqi1lCp6B6YXGNcop4T6gjXk1o2KJGFV2nm2Y8C53UXbK1UjFsHhG+ORWFRFbGiNrwsReMHamId9/YKOQ4pk3kMKX5HMC3kEO25FSbLvRON4IeTchIRCPWjOZse5MC2wQYENFOpgXlPC4gEVmfetAdql5SCwXwiLwW6Icvc1pQwtZk1xfBzzFsnawqlHLFX9J9Uz5J2b29uNXOoH9FrSTcFUorJ/3jHKBsV+ep7htnn53+A2WQWIGoLe5Drwu6xg5nbxvjVuk7TPe40061TXEvW9fU8M0R8EaLBMJSE/ascFKhFaqBHacTftkvxw7Cueg8stTIvQVo/QSpPQgWWJYZAeoUVCgZ64ciPBeJzp+HDUpq52O+df5ZO7HvTrjsJ1f4kMNIdDt9rGblv9FwfGJyLz3hvs/SZ/walIzNmlMHfxoaAYp7h52WJ6pMbIp8aQz4O+vHJvf0YzZwG7dv05I5IiTY/+ZlnsukaaEjrHqhLgsKKUQh76fxA51Hwrc/JtB7EYdGRzNmNcXftm+mCBd53pmfhPB55wFuh94K2Q9jfw2kF8mcDbxfhPAAAA//+UTfJd" } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json index 98a48adecbb..b3164786d1e 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -2,6 +2,7 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "awsfargate": { "task_stats": { + "cluster_name": "default", "cpu": { "core": null, "kernel": { @@ -61,6 +62,7 @@ }, "writes": 0 }, + "identifier": "query-metadata/1234", "memory": { "fail": { "count": 0 @@ -125,11 +127,15 @@ "packets": 25857 } } - } + }, + "task_name": "query-metadata" } }, + "cloud": { + "region": "us-west-2" + }, "container": { - "id": "query-metadata-1", + "id": "1234", "image": { "name": "mreferre/eksutils" }, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index e0099191f35..bcdaa3d4dac 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -4,6 +4,10 @@ `task_stats` contains the metrics that were scraped from AWS fargate task stats ${ECS_CONTAINER_METADATA_URI_V4}/task/stats metadata endpoint. release: beta fields: + - name: identifier + type: keyword + description: > + Container identifier across tasks and clusters, which equals to container.name + '/' + container.id. - name: cpu type: group description: Runtime CPU metrics. diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json index 65219722319..7c3d4a3e1c9 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -4,7 +4,7 @@ "Family": "query-metadata-1", "Revision": "7", "Containers": [{ - "DockerId": "query-metadata-1", + "DockerId": "1234", "Name": "query-metadata", "Image": "mreferre/eksutils", "Labels": { diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json index 88ddceeb6b4..f2d7bd36f86 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json @@ -1,5 +1,5 @@ { - "query-metadata-1": { + "1234": { "blkio_stats": { "io_service_bytes_recursive": [ {"major": 202, "minor": 26368, "op": "Read", "value": 3452928}, @@ -36,7 +36,7 @@ "throttled_time": 0 } }, - "id": "query-metadata-1", + "id": "1234", "memory_stats": { "limit": 3937787904, "max_usage": 15294464, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index ca8cca5dca8..79e63664bfb 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -5,12 +5,20 @@ package task_stats import ( + "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) +var ( + clusterLabel = "com_amazonaws_ecs_cluster" + taskLabel = "com_amazonaws_ecs_task-definition-family" +) + func eventsMapping(r mb.ReporterV2, statsList []Stats) { for _, stats := range statsList { r.Event(createEvent(&stats)) @@ -18,9 +26,8 @@ func eventsMapping(r mb.ReporterV2, statsList []Stats) { } func createEvent(stats *Stats) mb.Event { - return mb.Event{ - Timestamp: time.Time(stats.Time), - RootFields: createContainerFields(stats), + e := mb.Event{ + Timestamp: time.Time(stats.Time), MetricSetFields: common.MapStr{ "cpu": createCPUFields(stats), "memory": createMemoryFields(stats), @@ -28,10 +35,44 @@ func createEvent(stats *Stats) mb.Event { "diskio": createDiskIOFields(stats), }, } + + regionName, clusterName := getRegionAndClusterName(stats.Container.Labels) + e.RootFields = createRootFields(stats, regionName) + if clusterName != "" { + e.MetricSetFields.Put("cluster_name", clusterName) + } + + taskName := stats.Container.Labels[taskLabel] + if taskName != "" { + e.MetricSetFields.Put("task_name", taskName) + } + + e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) + return e } -func createContainerFields(stats *Stats) common.MapStr { - return common.MapStr{ +func generateIdentifier(containerName string, containerID string) string { + return containerName + "/" + containerID +} + +func getRegionAndClusterName(labels map[string]string) (regionName string, clusterName string) { + if v, ok := labels[clusterLabel]; ok { + vSplit := strings.Split(v, "cluster/") + if len(vSplit) == 2 { + clusterName = vSplit[1] + } + + arnParsed, err := arn.Parse(v) + if err == nil { + regionName = arnParsed.Region + } + return + } + return +} + +func createRootFields(stats *Stats, regionName string) common.MapStr { + rootFields := common.MapStr{ "container": common.MapStr{ "id": stats.Container.DockerId, "image": common.MapStr{ @@ -41,6 +82,15 @@ func createContainerFields(stats *Stats) common.MapStr { "labels": stats.Container.Labels, }, } + + // add cloud.region + if regionName != "" { + cloud := common.MapStr{ + "region": regionName, + } + rootFields.Put("cloud", cloud) + } + return rootFields } func createCPUFields(stats *Stats) common.MapStr { diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index 7193f640a63..67129f38ff7 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -30,6 +30,7 @@ func TestData(t *testing.T) { "metricsets": []string{"task_stats"}, } + os.Setenv("ECS_CONTAINER_METADATA_URI_V4", "1.2.3.4") m := mbtest.NewFetcher(t, config) taskStatsFile, err := os.Open("./_meta/testdata/task_stats.json") diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go index c54c3d5efdb..2ab2e5f934d 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -122,3 +122,15 @@ func TestGetCPUStats(t *testing.T) { assert.Equal(t, 0.4, cpuStats.TotalUsage) assert.Equal(t, 0.2, cpuStats.TotalUsageNormalized) } + +func TestGetRegionAndClusterName(t *testing.T) { + labels := map[string]string{} + labels["com_amazonaws_ecs_cluster"] = "arn:aws:ecs:us-east-1:1234:cluster/metricbeat-fargate-1" + regionName, clusterName := getRegionAndClusterName(labels) + assert.Equal(t, "us-east-1", regionName) + assert.Equal(t, "metricbeat-fargate-1", clusterName) +} + +func TestGenerateIdentifier(t *testing.T) { + assert.Equal(t, "container-name/123", generateIdentifier("container-name", "123")) +}