From 48d279414af9445dbc2977a1b0ab72ad0f74602b Mon Sep 17 00:00:00 2001 From: MakoWish <26614684+MakoWish@users.noreply.github.com> Date: Mon, 19 Jun 2023 18:30:07 +0300 Subject: [PATCH] ti_recordedfuture: ensure event.kind is correctly set for pipeline errors --- packages/aws/changelog.yml | 5 + ...-98f85120-0ea4-11ee-9c37-e55025c0278a.json | 2286 +++++++++++++++++ packages/aws/manifest.yml | 2 +- .../ingest_pipeline/decode_csv.yml | 7 + .../elasticsearch/ingest_pipeline/default.yml | 5 +- 5 files changed, 2303 insertions(+), 2 deletions(-) create mode 100644 packages/aws/kibana/dashboard/aws-98f85120-0ea4-11ee-9c37-e55025c0278a.json diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index cfdac90b767..c6a6ec27965 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.45.2" + changes: + - description: Add AWS EMR metrics dashboard. + type: enhancement + link: https://github.com/elastic/integrations/pull/6606 - version: "1.45.1" changes: - description: Add AWS API Gateway dashboard. diff --git a/packages/aws/kibana/dashboard/aws-98f85120-0ea4-11ee-9c37-e55025c0278a.json b/packages/aws/kibana/dashboard/aws-98f85120-0ea4-11ee-9c37-e55025c0278a.json new file mode 100644 index 00000000000..fcc4515d1a4 --- /dev/null +++ b/packages/aws/kibana/dashboard/aws-98f85120-0ea4-11ee-9c37-e55025c0278a.json @@ -0,0 +1,2286 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"939bbdd4-5843-4f44-bd0a-cc15f4f4efe8\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":false,\"width\":\"medium\",\"explicitInput\":{\"id\":\"939bbdd4-5843-4f44-bd0a-cc15f4f4efe8\",\"fieldName\":\"cloud.region\",\"title\":\"AWS Region\",\"enhancements\":{}}},\"a5ba1a06-889c-4758-b15b-b22263411798\":{\"type\":\"optionsListControl\",\"order\":1,\"grow\":false,\"width\":\"medium\",\"explicitInput\":{\"id\":\"a5ba1a06-889c-4758-b15b-b22263411798\",\"fieldName\":\"aws.dimensions.JobFlowId\",\"title\":\"Job Flow ID\",\"enhancements\":{}}}}" + }, + "description": "Overview of AWS EMR Metrics", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "HDFSUtilization", + "operationType": "average", + "params": { + "emptyAsNull": true, + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.HDFSUtilization.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "showBar": false + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "927fa50c-bf7d-4f7b-8679-518aa7f51262", + "w": 6, + "x": 0, + "y": 0 + }, + "panelIndex": "927fa50c-bf7d-4f7b-8679-518aa7f51262", + "title": "HDFS Utilization", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Apps Failed", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.AppsFailed.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "ef158595-61d2-4e7e-a55a-b6fd0fa8214d", + "w": 6, + "x": 6, + "y": 0 + }, + "panelIndex": "ef158595-61d2-4e7e-a55a-b6fd0fa8214d", + "title": "Apps Failed", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.S3BytesRead.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.S3BytesRead.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "c3ce6c26-e55a-44e2-907c-1d2aab3529b7", + "w": 36, + "x": 12, + "y": 0 + }, + "panelIndex": "c3ce6c26-e55a-44e2-907c-1d2aab3529b7", + "title": "S3 Bytes Read", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "YARN Memory Available Percentage", + "operationType": "average", + "params": { + "emptyAsNull": true, + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.YARNMemoryAvailablePercentage.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "9e85d781-6cf2-4e95-ac2f-385711c74765", + "w": 6, + "x": 0, + "y": 5 + }, + "panelIndex": "9e85d781-6cf2-4e95-ac2f-385711c74765", + "title": "YARN Memory Available Percentage", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Apps Completed", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.AppsCompleted.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "53377d5a-7df3-4634-9300-293ce2fbbd64", + "w": 6, + "x": 6, + "y": 5 + }, + "panelIndex": "53377d5a-7df3-4634-9300-293ce2fbbd64", + "title": "Apps Completed", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Container Pending Ratio", + "operationType": "average", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.ContainerPendingRatio.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "19d0e843-0006-4b64-b917-42e38c5f7e75", + "w": 6, + "x": 0, + "y": 10 + }, + "panelIndex": "19d0e843-0006-4b64-b917-42e38c5f7e75", + "title": "Container Pending Ratio", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Apps Killed", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.AppsKilled.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "metricAccessor": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "b9e76ae0-d057-4ff7-900d-aee86b4b32ec", + "w": 6, + "x": 6, + "y": 10 + }, + "panelIndex": "b9e76ae0-d057-4ff7-900d-aee86b4b32ec", + "title": "Apps Killed", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.S3BytesWritten.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.S3BytesWritten.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "fc0a7c7e-c30a-4777-b344-39bf15c70c45", + "w": 24, + "x": 0, + "y": 15 + }, + "panelIndex": "fc0a7c7e-c30a-4777-b344-39bf15c70c45", + "title": "S3 Bytes Written", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.HDFSBytesWritten.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.HDFSBytesWritten.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "dc9d2105-5101-498d-9b75-44e95382c477", + "w": 24, + "x": 24, + "y": 15 + }, + "panelIndex": "dc9d2105-5101-498d-9b75-44e95382c477", + "title": "HDFS Bytes Written", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.HDFSBytesRead.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.HDFSBytesRead.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "c16dc181-d15d-4301-9bf3-95dddd75a64b", + "w": 24, + "x": 0, + "y": 30 + }, + "panelIndex": "c16dc181-d15d-4301-9bf3-95dddd75a64b", + "title": "HDFS Bytes Read", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.MemoryTotalMB.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.MemoryTotalMB.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "032f8c5d-1580-4b7f-b3f2-fc075533a51b", + "w": 24, + "x": 24, + "y": 30 + }, + "panelIndex": "032f8c5d-1580-4b7f-b3f2-fc075533a51b", + "title": "Memory Total MB", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.elasticmapreduce.metrics.CoreNodesRunning.avg", + "operationType": "average", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.CoreNodesRunning.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "3832f35d-690e-4ec8-bcd8-6c808b1fc277", + "w": 24, + "x": 0, + "y": 45 + }, + "panelIndex": "3832f35d-690e-4ec8-bcd8-6c808b1fc277", + "title": "Core Nodes Running", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.elasticmapreduce.metrics.TotalNodesRunning.avg", + "operationType": "average", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.TotalNodesRunning.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "fbe53db9-c526-4b2c-a2d5-0a76a301d466", + "w": 24, + "x": 24, + "y": 45 + }, + "panelIndex": "fbe53db9-c526-4b2c-a2d5-0a76a301d466", + "title": "Total Nodes Running", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.elasticmapreduce.metrics.TaskNodesRunning.avg", + "operationType": "average", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.TaskNodesRunning.avg" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "d2541523-f58c-485a-9bea-0899d9eb95b4", + "w": 24, + "x": 0, + "y": 60 + }, + "panelIndex": "d2541523-f58c-485a-9bea-0899d9eb95b4", + "title": "Task Nodes Running", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.MemoryAvailableMB.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.MemoryAvailableMB.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "1f421921-3327-443f-8bfc-01645f331bdc", + "w": 24, + "x": 24, + "y": 60 + }, + "panelIndex": "1f421921-3327-443f-8bfc-01645f331bdc", + "title": "Memory Available MB", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.MemoryReservedMB.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.MemoryReservedMB.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "curveType": "LINEAR", + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "a1f95551-a9a4-44c6-afa2-3b43e23df9e4", + "w": 24, + "x": 0, + "y": 75 + }, + "panelIndex": "a1f95551-a9a4-44c6-afa2-3b43e23df9e4", + "title": "Memory Reserved MB", + "type": "lens", + "version": "8.7.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "775c326f-9e29-49ea-a6f9-aab9fb4ee280": { + "columnOrder": [ + "2aaf65a9-0922-4200-9704-740e9132668e", + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109", + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "columns": { + "2aaf65a9-0922-4200-9704-740e9132668e": { + "dataType": "string", + "isBucketed": true, + "label": "Top 10 values of aws.dimensions.JobFlowId", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.JobFlowId" + }, + "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "60s" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of aws.elasticmapreduce.metrics.MemoryAllocatedMB.sum", + "operationType": "sum", + "params": { + "emptyAsNull": true, + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "aws.elasticmapreduce.metrics.MemoryAllocatedMB.sum" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "textBased": { + "layers": {} + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f1c1b021-d02d-4e4a-bd4d-7b8a5a10adb6" + ], + "layerId": "775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "layerType": "data", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "2aaf65a9-0922-4200-9704-740e9132668e", + "xAccessor": "83ef5d76-6b8c-49e6-b6c6-e0e3604d6109" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "1785d7d7-8a43-4a93-a550-1590b005b31c", + "w": 24, + "x": 24, + "y": 75 + }, + "panelIndex": "1785d7d7-8a43-4a93-a550-1590b005b31c", + "title": "Memory Allocated MB", + "type": "lens", + "version": "8.7.1" + } + ], + "timeRestore": false, + "title": "[Metrics AWS] EMR Overview", + "version": 1 + }, + "coreMigrationVersion": "8.7.1", + "created_at": "2023-06-19T14:28:24.447Z", + "id": "aws-98f85120-0ea4-11ee-9c37-e55025c0278a", + "migrationVersion": { + "dashboard": "8.7.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "927fa50c-bf7d-4f7b-8679-518aa7f51262:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "ef158595-61d2-4e7e-a55a-b6fd0fa8214d:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "c3ce6c26-e55a-44e2-907c-1d2aab3529b7:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "9e85d781-6cf2-4e95-ac2f-385711c74765:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "53377d5a-7df3-4634-9300-293ce2fbbd64:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "19d0e843-0006-4b64-b917-42e38c5f7e75:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "b9e76ae0-d057-4ff7-900d-aee86b4b32ec:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "fc0a7c7e-c30a-4777-b344-39bf15c70c45:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "dc9d2105-5101-498d-9b75-44e95382c477:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "c16dc181-d15d-4301-9bf3-95dddd75a64b:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "032f8c5d-1580-4b7f-b3f2-fc075533a51b:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "3832f35d-690e-4ec8-bcd8-6c808b1fc277:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "fbe53db9-c526-4b2c-a2d5-0a76a301d466:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "d2541523-f58c-485a-9bea-0899d9eb95b4:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "1f421921-3327-443f-8bfc-01645f331bdc:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "a1f95551-a9a4-44c6-afa2-3b43e23df9e4:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "1785d7d7-8a43-4a93-a550-1590b005b31c:indexpattern-datasource-layer-775c326f-9e29-49ea-a6f9-aab9fb4ee280", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "controlGroup_939bbdd4-5843-4f44-bd0a-cc15f4f4efe8:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "controlGroup_a5ba1a06-889c-4758-b15b-b22263411798:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index adc533327bc..43a322f6a2e 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.45.1 +version: 1.45.2 license: basic description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/decode_csv.yml b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/decode_csv.yml index 86c06b7a1df..ced97863c6b 100644 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/decode_csv.yml +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/decode_csv.yml @@ -41,3 +41,10 @@ processors: ctx['json'] = dst; - remove: field: _tmp_ +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index 24a77c4f5f2..1d477017728 100644 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -240,6 +240,9 @@ processors: - _temp_ ignore_missing: true on_failure: + - set: + field: event.kind + value: pipeline_error - append: field: error.message - value: "{{{ _ingest.on_failure_message }}}" + value: '{{{ _ingest.on_failure_message }}}'