diff --git a/packages/awsfirehose/_dev/build/build.yml b/packages/awsfirehose/_dev/build/build.yml index 47cbed9fed8..e2b012548e0 100644 --- a/packages/awsfirehose/_dev/build/build.yml +++ b/packages/awsfirehose/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@v8.0.0 + reference: git@v8.11.0 diff --git a/packages/awsfirehose/_dev/build/docs/README.md b/packages/awsfirehose/_dev/build/docs/README.md index 72af9cdfba1..1d45bc7c91e 100644 --- a/packages/awsfirehose/_dev/build/docs/README.md +++ b/packages/awsfirehose/_dev/build/docs/README.md @@ -1,8 +1,9 @@ # Amazon Data Firehose -Amazon Data Firehose integration offers users a way to stream logs from Firehose to Elastic Cloud. -This integration includes predefined rules that automatically route AWS service logs to the respective integrations, which -include field mappings, ingest pipelines, predefined dashboards and ect. Here is a list of log types that are supported -by this integration: +Amazon Data Firehose integration offers users a way to stream logs and CloudWatch metrics from Firehose to Elastic Cloud. +This integration includes predefined rules that automatically route AWS service logs and CloudWatch metrics to the respective integrations, which +include field mappings, ingest pipelines, and predefined dashboards. + +Here is a list of log types that are supported by this integration: | AWS service log | Log destination | |--------------------|--------------------------| @@ -17,6 +18,31 @@ by this integration: | VPC Flow | Firehose, CloudWatch, S3 | | WAF | Firehose, CloudWatch. S3 | +Here is a list of CloudWatch metrics that are supported by this integration: + +| AWS service monitoring metrics | +|--------------------------------| +| API Gateway | +| DynamoDB | +| EBS | +| EC2 | +| ECS | +| ELB | +| EMR | +| Network Firewall | +| Kafka | +| Kinesis | +| Lambda | +| NATGateway | +| RDS | +| S3 | +| S3 Storage Lens | +| SNS | +| SQS | +| TransitGateway | +| Usage | +| VPN | + ## Limitation It is not possible to configure a delivery stream to send data to Elastic Cloud via PrivateLink (VPC endpoint). This is a current limitation in Firehose, which we are working with AWS to resolve. @@ -91,6 +117,12 @@ This is a current limitation in Firehose, which we are working with AWS to resol This parameter will increase the data volume in Elasticsearch and should be used with care. 3. Send data to the Firehose delivery stream - + 1. logs Consult the [AWS documentation](https://docs.aws.amazon.com/firehose/latest/dev/basic-write.html) for details on how to configure a variety of log sources to send data to Firehose delivery streams. + + 2. metrics + Consult the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup.html) + for details on how to set up a metric stream in CloudWatch and + [Custom setup with Firehose](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup-datalake.html) + to send metrics to Firehose. For Elastic, we only support JSON and OpenTelemetry 1.0.0 formats for the metrics. diff --git a/packages/awsfirehose/changelog.yml b/packages/awsfirehose/changelog.yml index 6b971895324..2e9d5d11ed0 100644 --- a/packages/awsfirehose/changelog.yml +++ b/packages/awsfirehose/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Add routing rules for metrics from Firehose. + type: enhancement + link: https://github.com/elastic/integrations/pull/9916 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-apigateway-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-apigateway-log.json-expected.json index da13eb2384b..0eb388a76f7 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-apigateway-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-apigateway-log.json-expected.json @@ -9,9 +9,6 @@ "aws.firehose.subscription_filters": "[apigateway-to-firehose]", "aws.kinesis.name": "firehose-apigateway-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -19,7 +16,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "{\"requestId\":\"GQIVriFLIAMEMsA=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"09/Jun/2023:12:54:08 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"47140\"}" @@ -33,9 +30,6 @@ "aws.firehose.subscription_filters": "[apigateway-to-firehose]", "aws.kinesis.name": "firehose-apigateway-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -43,7 +37,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "{\"requestId\":\"Iq9gjE_aIAMFZTg=\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:20:44 +0000\",\"eventType\":\"CONNECT\",\"routeKey\":\"$connect\",\"status\":\"500\",\"connectionId\":\"Iq8gj1UmIAMCKpA=\",\"apiId\":\"z1ctxygne5\",\"stage\":\"production\",\"domainName\":\"z1ctxygne5.execute-api.us-east-1.amazonaws.com\"}" @@ -57,9 +51,6 @@ "aws.firehose.subscription_filters": "[apigateway-to-firehose]", "aws.kinesis.name": "firehose-apigateway-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -67,7 +58,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "{\"requestId\":\"48752d0f-c99d-4cfa-a5a7-f3c6834d19e5\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"10/Jun/2023:15:36:28 +0000\",\"httpMethod\":\"GET\",\"resourcePath\":\"/pets\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"184\"}" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json index 2650a960dbb..0f9be13bfbc 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json @@ -1,19 +1,19 @@ { - "events": [ - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-cloudfront-logs-to-elastic", - "data_stream.namespace": "default", - "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - } - ] + "events": [ + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-cloudfront-logs-to-elastic", + "data_stream.namespace": "default", + "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + } + ] } \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json-expected.json index e5e541b2700..bac454ba713 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudfront-log.json-expected.json @@ -6,9 +6,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -16,7 +13,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudtrail-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudtrail-log.json-expected.json index ff73078498f..31948a87721 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudtrail-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-cloudtrail-log.json-expected.json @@ -9,9 +9,6 @@ "aws.firehose.subscription_filters": "[cloudtrail-to-firehose]", "aws.kinesis.name": "firehose-cloudtrail-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-2", @@ -19,7 +16,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AWSService\",\"invokedBy\":\"cloudtrail.amazonaws.com\"},\"eventTime\":\"2023-07-17T21:02:26Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"sa-east-1\",\"sourceIPAddress\":\"cloudtrail.amazonaws.com\",\"userAgent\":\"cloudtrail.amazonaws.com\",\"requestParameters\":{\"roleArn\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\",\"roleSessionName\":\"CLOUDWATCH_LOGS_DELIVERY_SESSION\"},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAZEDJODE3A5LVGLFB\",\"sessionToken\":\"IQoJb3JpZ2luX2VjEGUaCXNhLWVhc3QtMSJHMEUCIHgHmtcrhwDhosJlQVky+C2zsYDKuR99qVlNjGIp8FLWAiEAsJtTDQ3Arq8iXEOHwv0ImEQdGb5tbgc+fLpoK58Enb4q9AII3v//////////ARAEGgw2MjcyODYzNTAxMzQiDN5gNdfO4ZdSqDmmwSrIAicTBYZg+ZXjwiJTN/Bz2YsMWYU6psw5znG3/Gh3EJ1P3RCmB7d79X6XZzFVi2u2xdrnaY/sTKDfp1jdl8OoAsSKYwJiGbzjoQlv59bB6JqPbKfAKUPAmz6JEMWNFgWTtaQL9rNkdPz23u/1msoUSzxCcxR9f3A2dD4yqnVpNJe8ipuhxpBMzQ61vcGL4G5hQEDM/o8sORP2PXbK4O7QAuWOyuryYkHAPwY9RrL0WHfflGBEBQV6XlidGpsRCtIppZVn025n3DQOypDEaL3fKp0gUsMkDH+frFjxop4o4wRYC3CxXe3XRJ5/Te886rQry7RUfXlQtiCfojZO5ohcLB+z6Y/uCK0IHp3zrfl5shKsQIAFt7p0B8W7PK5yHE4W9HHRiktJ9wTtq1YCTaWECpnjW0bISNgumRmDOAJvVHAjSjfkr4yAlJkw4qm8pQY6vwGbBiuf98AfRFrXMy01hVdE3GNTBrIS68zxUJaOjBLgw8l0nEC00L+LPuqaASFWz65Dnq5JAjXaDD9E3iCi4klp4gZFAcj7uGgeBIPkP7Bpr4SvBfnnqCgE2oyFrWke3NnYtqkL5iHLJeGlOTrvI5ND2H4jurQv0KbiqwHt6DmGF3poZOrtf8R3piNcuCCDLU8RvhRVLHy5rKPzsWgNokBc9XXmgltwvB6rIgdZhBJzupzmy/NSoWZcOeH2ooEELw==\",\"expiration\":\"Jul 12, 2023, 10:02:26 PM\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAZEDJODE3NLJAH2FZC:CLOUDWATCH_LOGS_DELIVERY_SESSION\",\"arn\":\"arn:aws:sts::123456:assumed-role/aws-cloudtrail-logs-123456-b888baff_Role/CLOUDWATCH_LOGS_DELIVERY_SESSION\"}},\"requestID\":\"041c9e5f-a031-47d2-a4a0-011bc8d5352c\",\"eventID\":\"3096b662-7aa9-43e6-8bee-541a45686745\",\"readOnly\":true,\"resources\":[{\"accountId\":\"123456\",\"type\":\"AWS::IAM::Role\",\"ARN\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\"}],\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"123456\",\"sharedEventID\":\"a1c94275-884f-4c1f-b8dc-2e1bf4c94d29\",\"eventCategory\":\"Management\"}" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json index 5a5b411a9d5..53bb3122a83 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json @@ -1,49 +1,49 @@ { - "events": [ - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-classic-load-balancer-logs-to-elastic", - "data_stream.namespace": "default", - "message": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-classic-load-balancer-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - }, - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-application-load-balancer-logs-to-elastic", - "data_stream.namespace": "default", - "message": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-application-load-balancer-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - }, - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-network-load-balancer-logs-to-elastic", - "data_stream.namespace": "default", - "message": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-network-load-balancer-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - } - ] + "events": [ + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-classic-load-balancer-logs-to-elastic", + "data_stream.namespace": "default", + "message": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-classic-load-balancer-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + }, + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-application-load-balancer-logs-to-elastic", + "data_stream.namespace": "default", + "message": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-application-load-balancer-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + }, + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-network-load-balancer-logs-to-elastic", + "data_stream.namespace": "default", + "message": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-network-load-balancer-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + } + ] } \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json-expected.json index b6443db45cc..0d4be588b73 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-elb-log.json-expected.json @@ -6,9 +6,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-classic-load-balancer-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -16,7 +13,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -" @@ -27,9 +24,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-application-load-balancer-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -37,7 +31,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"" @@ -48,9 +42,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-network-load-balancer-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -58,7 +49,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", "message": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-firewall-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-firewall-log.json-expected.json index 677a47d4067..cd23541ebcb 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-firewall-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-firewall-log.json-expected.json @@ -7,9 +7,6 @@ "aws.firehose.subscription_filters": "[test-firewall-logs-to-firehose]", "aws.kinesis.name": "test-firewall-logs", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456789", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -17,7 +14,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37728046078123216000395549868459931814660237705210691585", "message": "{\"firewall_name\":\"AWSNetworkFirewall\",\"availability_zone\":\"us-east-2a\",\"event_timestamp\":\"1636381332\",\"event\":{\"timestamp\":\"2021-11-08T14:22:12.637611+0000\",\"flow_id\":706471429191862,\"event_type\":\"alert\",\"src_ip\":\"81.2.69.143\",\"src_port\":51254,\"dest_ip\":\"216.160.83.57\",\"dest_port\":80,\"proto\":\"TCP\",\"alert\":{\"action\":\"blocked\",\"signature_id\":1000003,\"rev\":1,\"signature\":\"Deny all other TCP traffic\",\"category\":\"\",\"severity\":3},\"http\":{\"hostname\":\"216.160.83.57\",\"url\":\"/\",\"http_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"length\":0},\"app_proto\":\"http\"}}" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-public-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-public-log.json-expected.json index d3308f579c1..fd9a2052a68 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-public-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-public-log.json-expected.json @@ -9,9 +9,6 @@ "aws.firehose.subscription_filters": "[test-route53-public-logs-to-firehose]", "aws.kinesis.name": "test-route53-public-logs", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456789", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -19,7 +16,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37728046078123216000395549868459931814660237705210691585", "message": "1.0 2023-08-11T20:01:37Z Z0786514BU8K9GJ587CT filebeat-firehose.com NAPTR NOERROR UDP EWR52-C2 44.199.191.178 -" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-resolver-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-resolver-log.json-expected.json index d1838501d88..de4c00d416c 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-resolver-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-route53-resolver-log.json-expected.json @@ -9,9 +9,6 @@ "aws.firehose.subscription_filters": "[test-route53-resolver-logs-to-firehose]", "aws.kinesis.name": "test-route53-resolver-logs", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456789", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -19,7 +16,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37728046078123216000395549868459931814660237705210691585", "message": "{\"version\":\"1.100000\",\"account_id\":\"626345947581\",\"region\":\"us-east-1\",\"vpc_id\":\"vpc-01e31a7c\",\"query_timestamp\":\"2021-12-11T22:46:23Z\",\"query_name\":\"does-not-exist.abc.com.\",\"query_type\":\"A\",\"query_class\":\"IN\",\"rcode\":\"NXDOMAIN\",\"answers\":[],\"srcaddr\":\"172.31.86.159\",\"srcport\":\"48701\",\"transport\":\"UDP\",\"srcids\":{}}" diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json index 534d8c007be..49493f3cc31 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json @@ -1,34 +1,34 @@ { - "events": [ - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic", - "data_stream.namespace": "default", - "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-s3access-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - }, - { - "cloud.region": "us-east-1", - "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic", - "data_stream.namespace": "default", - "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader - TLSv1.2", - "aws.kinesis.type": "deliverystream", - "data_stream.type": "logs", - "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", - "cloud.provider": "aws", - "@timestamp": "2023-07-25T21:04:35Z", - "cloud.account.id": "123456", - "data_stream.dataset": "awsfirehose", - "aws.kinesis.name": "firehose-s3access-logs-to-elastic", - "event.id": "37670326805251200781477669690942747782212394134076063744" - } - ] + "events": [ + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic", + "data_stream.namespace": "default", + "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-s3access-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + }, + { + "cloud.region": "us-east-1", + "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic", + "data_stream.namespace": "default", + "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader - TLSv1.2", + "aws.kinesis.type": "deliverystream", + "data_stream.type": "logs", + "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", + "cloud.provider": "aws", + "@timestamp": "2023-07-25T21:04:35Z", + "cloud.account.id": "123456", + "data_stream.dataset": "awsfirehose", + "aws.kinesis.name": "firehose-s3access-logs-to-elastic", + "event.id": "37670326805251200781477669690942747782212394134076063744" + } + ] } diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json index 301b2f23db9..d791660ef73 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json @@ -6,9 +6,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-s3access-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -16,10 +13,10 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", - "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2" + "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2" }, { "@timestamp": "2023-07-25T21:04:35Z", @@ -27,9 +24,6 @@ "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55", "aws.kinesis.name": "firehose-s3access-logs-to-elastic", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -37,10 +31,10 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37670326805251200781477669690942747782212394134076063744", - "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader - TLSv1.2" + "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader - TLSv1.2" } ] } \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-vpcflow-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-vpcflow-log.json-expected.json index 1c07807b08a..662850ac20a 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-vpcflow-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-vpcflow-log.json-expected.json @@ -6,9 +6,6 @@ "aws.firehose.request_id": "1cfbed13-d631-4b8b-b20a-b7c5bf8fcd00", "aws.kinesis.name": "test-vpcflow-logs", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "428152502467", "cloud.provider": "aws", "cloud.region": "us-east-2", @@ -16,7 +13,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "message": "{\"message\":\"2 428152502467 eni-0b584e1c714317ac6 176.111.174.91 10.0.0.102 41536 1135 6 1 40 1692809104 1692809162 REJECT OK\"}\n" } diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-waf-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-waf-log.json-expected.json index 82037490543..393f11c9fc4 100644 --- a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-waf-log.json-expected.json +++ b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-waf-log.json-expected.json @@ -6,9 +6,6 @@ "aws.firehose.request_id": "1cfbed13-d631-4b8b-b20a-b7c5bf8fcd00", "aws.kinesis.name": "aws-waf-logs-test", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "428152502467", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -16,7 +13,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "message": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"x-stm-test\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}\n" }, @@ -29,9 +26,6 @@ "aws.firehose.subscription_filters": "[test-waf-logs-to-firehose]", "aws.kinesis.name": "test-waf-logs", "aws.kinesis.type": "deliverystream", - "cloud": { - "provider": "aws" - }, "cloud.account.id": "123456789", "cloud.provider": "aws", "cloud.region": "us-east-1", @@ -39,7 +33,7 @@ "data_stream.namespace": "default", "data_stream.type": "logs", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event.id": "37728046078123216000395549868459931814660237705210691585", "message": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"x-stm-test\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}\n" diff --git a/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml index cbcb28989ba..cba13e11e8a 100644 --- a/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml @@ -3,10 +3,7 @@ description: Pipeline for rerouting logs streams from Amazon Data Firehose. processors: - set: field: ecs.version - value: 8.0.0 - - set: - field: cloud.provider - value: aws + value: 8.11.0 on_failure: - set: field: error.message diff --git a/packages/awsfirehose/data_stream/logs/fields/ecs.yml b/packages/awsfirehose/data_stream/logs/fields/ecs.yml index c86ca327c7a..c2fbf7e6e5a 100644 --- a/packages/awsfirehose/data_stream/logs/fields/ecs.yml +++ b/packages/awsfirehose/data_stream/logs/fields/ecs.yml @@ -1,24 +1,8 @@ -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region - external: ecs name: data_stream.type - external: ecs name: data_stream.dataset - external: ecs name: data_stream.namespace -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: message - external: ecs name: '@timestamp' diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json new file mode 100644 index 00000000000..dd2bf1c760a --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json @@ -0,0 +1,26 @@ +{ + "events": [ + { + "@timestamp": "2024-05-01T08:24:00Z", + "start_timestamp": "2024-05-01T08:23:00Z", + "agent.type": "firehose", + "cloud.provider": "aws", + "cloud.account.id": "12345678901", + "cloud.region": "ap-southeast-2", + "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics", + "aws.cloudwatch.namespace": "AWS/ApiGateway", + "aws.dimensions": { + "ApiId": "6am7mj7jqx" + }, + "aws.apigateway.metrics.4XXError": { + "count": 1, + "sum": 0.0, + "min": 0.0, + "max": 0.0 + }, + "data_stream.type": "metrics", + "data_stream.dataset": "aws.cloudwatch", + "data_stream.namespace": "default" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json new file mode 100644 index 00000000000..bd5e2ed148a --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json @@ -0,0 +1,29 @@ +{ + "expected": [ + { + "@timestamp": "2024-05-01T08:24:00Z", + "agent.type": "firehose", + "aws.apigateway.metrics.4XXError": { + "count": 1, + "max": 0.0, + "min": 0.0, + "sum": 0.0 + }, + "aws.cloudwatch.namespace": "AWS/ApiGateway", + "aws.dimensions": { + "ApiId": "6am7mj7jqx" + }, + "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics", + "cloud.account.id": "12345678901", + "cloud.provider": "aws", + "cloud.region": "ap-southeast-2", + "data_stream.dataset": "aws.apigateway_metrics", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "8.11.0" + }, + "start_timestamp": "2024-05-01T08:23:00Z" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json new file mode 100644 index 00000000000..04ae70ab6b7 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json @@ -0,0 +1,25 @@ +{ + "events": [ + { + "@timestamp": "2024-06-12T22:00:00.000Z", + "start_timestamp": "2024-06-12T21:59:00.000Z", + "agent.type": "firehose", + "cloud.provider": "aws", + "cloud.account.id": "627286350134", + "cloud.region": "us-east-1", + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:627286350134:metric-stream/test-all-metrics-ks-1", + "aws.cloudwatch.namespace": "AWS/RDS", + "aws.dimensions": { + "DBInstanceIdentifier": "metricbeat-test-019e245b" + }, + "aws.rds.metrics.FreeStorageSpace": { + "count": 1, + "sum": 18430775296, + "avg": 18430775296, + "min": 18430775296, + "max": 18430775296 + }, + "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json-expected.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json-expected.json new file mode 100644 index 00000000000..1d5e4e48fb6 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "@timestamp": "2024-06-12T22:00:00.000Z", + "agent.type": "firehose", + "aws.cloudwatch.namespace": "AWS/RDS", + "aws.dimensions": { + "DBInstanceIdentifier": "metricbeat-test-019e245b" + }, + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:627286350134:metric-stream/test-all-metrics-ks-1", + "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63", + "aws.rds.metrics.FreeStorageSpace": { + "avg": 18430775296, + "count": 1, + "max": 18430775296, + "min": 18430775296, + "sum": 18430775296 + }, + "cloud.account.id": "627286350134", + "cloud.provider": "aws", + "cloud.region": "us-east-1", + "data_stream": { + "dataset": "aws.rds", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "start_timestamp": "2024-06-12T21:59:00.000Z" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json new file mode 100644 index 00000000000..df032dc638d --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json @@ -0,0 +1,48 @@ +{ + "events": [ + { + "@timestamp": "2024-04-30T08:23:00Z", + "start_timestamp": "2024-05-01T08:23:00Z", + "agent.type": "firehose", + "cloud.provider": "aws", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "StorageType": "StandardStorage" + }, + "aws.s3.metrics.BucketSizeBytes": { + "count": 1, + "sum": 76353832, + "min": 76353832, + "max": 76353832 + }, + "data_stream.type": "metrics", + "data_stream.dataset": "aws.cloudwatch", + "data_stream.namespace": "default" + }, + { + "@timestamp": "2024-04-30T08:23:00Z", + "start_timestamp": "2024-05-01T08:23:00Z", + "agent.type": "firehose", + "cloud.provider": "aws", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "StorageType": "StandardStorage" + }, + "aws.s3.metrics.NumberOfObjects": { + "count": 1, + "sum": 5, + "min": 5, + "max": 5 + }, + "data_stream.type": "metrics", + "data_stream.dataset": "aws.cloudwatch", + "data_stream.namespace": "default" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json-expected.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json-expected.json new file mode 100644 index 00000000000..5700fa74c4f --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json-expected.json @@ -0,0 +1,54 @@ +{ + "expected": [ + { + "@timestamp": "2024-04-30T08:23:00Z", + "agent.type": "firehose", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "StorageType": "StandardStorage" + }, + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.s3.metrics.BucketSizeBytes": { + "count": 1, + "max": 76353832, + "min": 76353832, + "sum": 76353832 + }, + "cloud.account.id": "123456789012", + "cloud.provider": "aws", + "cloud.region": "us-east-1", + "data_stream.dataset": "aws.s3_daily_storage", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "8.11.0" + }, + "start_timestamp": "2024-05-01T08:23:00Z" + }, + { + "@timestamp": "2024-04-30T08:23:00Z", + "agent.type": "firehose", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "StorageType": "StandardStorage" + }, + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.s3.metrics.NumberOfObjects": { + "count": 1, + "max": 5, + "min": 5, + "sum": 5 + }, + "cloud.account.id": "123456789012", + "cloud.provider": "aws", + "cloud.region": "us-east-1", + "data_stream.dataset": "aws.s3_daily_storage", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "8.11.0" + }, + "start_timestamp": "2024-05-01T08:23:00Z" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json new file mode 100644 index 00000000000..dd3a9b8825b --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json @@ -0,0 +1,27 @@ +{ + "events": [ + { + "@timestamp": "2024-05-01T08:24:00Z", + "start_timestamp": "2024-05-01T08:23:00Z", + "agent.type": "firehose", + "cloud.provider": "aws", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "BucketName": "s3-bucket", + "StorageType": "StandardStorage" + }, + "aws.s3.metrics.AllRequests": { + "count": 1, + "sum": 30, + "min": 30, + "max": 30 + }, + "data_stream.type": "metrics", + "data_stream.dataset": "aws.cloudwatch", + "data_stream.namespace": "default" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json-expected.json b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json-expected.json new file mode 100644 index 00000000000..9dfe0ceb828 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json-expected.json @@ -0,0 +1,30 @@ +{ + "expected": [ + { + "@timestamp": "2024-05-01T08:24:00Z", + "agent.type": "firehose", + "aws.cloudwatch.namespace": "AWS/S3", + "aws.dimensions": { + "BucketName": "s3-bucket", + "StorageType": "StandardStorage" + }, + "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3", + "aws.s3.metrics.AllRequests": { + "count": 1, + "max": 30, + "min": 30, + "sum": 30 + }, + "cloud.account.id": "123456789012", + "cloud.provider": "aws", + "cloud.region": "us-east-1", + "data_stream.dataset": "aws.s3_request", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "8.11.0" + }, + "start_timestamp": "2024-05-01T08:23:00Z" + } + ] +} \ No newline at end of file diff --git a/packages/awsfirehose/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml b/packages/awsfirehose/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..09523365aa3 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for rerouting metrics stream from Amazon Data Firehose. +processors: + - set: + field: ecs.version + value: 8.11.0 +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/awsfirehose/data_stream/metrics/fields/ecs.yml b/packages/awsfirehose/data_stream/metrics/fields/ecs.yml new file mode 100644 index 00000000000..30e1e9fe81c --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/fields/ecs.yml @@ -0,0 +1,14 @@ +- external: ecs + name: cloud.account.id + dimension: true +- external: ecs + name: cloud.region + dimension: true +- external: ecs + name: data_stream.type +- external: ecs + name: data_stream.dataset +- external: ecs + name: data_stream.namespace +- external: ecs + name: '@timestamp' diff --git a/packages/awsfirehose/data_stream/metrics/fields/fields.yml b/packages/awsfirehose/data_stream/metrics/fields/fields.yml new file mode 100644 index 00000000000..b4b5d3864fd --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/fields/fields.yml @@ -0,0 +1,40 @@ +- name: start_timestamp + type: date + description: Date/time when the monitoring cycle started. +- name: aws + type: group + fields: + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. + dimension: true + - name: exporter + type: group + fields: + - name: arn + type: keyword + description: The metric stream ARN. + dimension: true + - name: dimensions.* + type: keyword + description: | + Metric dimensions. + dimension: true + - name: firehose + type: group + fields: + - name: parameters.* + type: flattened + description: | + Key-value pairs users set up when creating the Kinesis Data Firehose. These parameters are included in each HTTP call. +- name: aws.*.metrics.*.* + type: object + object_type: double + object_type_mapping_type: "*" + metric_type: gauge + subobjects: false + description: | + Metrics that returned from Cloudwatch API query. diff --git a/packages/awsfirehose/data_stream/metrics/manifest.yml b/packages/awsfirehose/data_stream/metrics/manifest.yml new file mode 100644 index 00000000000..12ab3298549 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/manifest.yml @@ -0,0 +1,9 @@ +title: Metrics ingested from Amazon Data Firehose +type: metrics +dataset: aws.cloudwatch +# This integration includes predefined rules that automatically route CloudWatch metrics ingested from Amazon Data Firehose to the respective integrations. +# Adding these permissions to ensure the agents have permissions to write data to `metrics-*-*`. +elasticsearch: + index_mode: "time_series" + dynamic_dataset: true + dynamic_namespace: true diff --git a/packages/awsfirehose/data_stream/metrics/routing_rules.yml b/packages/awsfirehose/data_stream/metrics/routing_rules.yml new file mode 100644 index 00000000000..57a8c53a391 --- /dev/null +++ b/packages/awsfirehose/data_stream/metrics/routing_rules.yml @@ -0,0 +1,107 @@ +- source_dataset: aws.cloudwatch + rules: + - target_dataset: aws.apigateway_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/ApiGateway" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.dynamodb + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/DynamoDB" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.ebs + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/EBS" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.ec2_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/EC2" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.ecs_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/ECS" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.elb_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/ELB" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.emr_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/ElasticMapReduce" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.firewall_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/NetworkFirewall" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.kafka_metrics + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/Kafka" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.kinesis + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/Kinesis" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.lambda + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/Lambda" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.natgateway + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/NATGateway" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.rds + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/RDS" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.s3_storage_lens + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/S3/Storage-Lens" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.sns + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/SNS" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.sqs + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/SQS" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.transitgateway + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/TransitGateway" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.usage + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/Usage" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.vpn + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/VPN" + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.s3_daily_storage + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/S3" && (ctx['aws.s3.metrics.BucketSizeBytes'] != null || ctx['aws.s3.metrics.NumberOfObjects'] != null) + namespace: + - "{{data_stream.namespace}}" + - default + - target_dataset: aws.s3_request + if: ctx['aws.cloudwatch.namespace'] != null && ctx['aws.cloudwatch.namespace'] == "AWS/S3" && ctx['aws.s3.metrics.BucketSizeBytes'] == null && ctx['aws.s3.metrics.NumberOfObjects'] == null + namespace: + - "{{data_stream.namespace}}" + - default diff --git a/packages/awsfirehose/docs/README.md b/packages/awsfirehose/docs/README.md index 72af9cdfba1..1d45bc7c91e 100644 --- a/packages/awsfirehose/docs/README.md +++ b/packages/awsfirehose/docs/README.md @@ -1,8 +1,9 @@ # Amazon Data Firehose -Amazon Data Firehose integration offers users a way to stream logs from Firehose to Elastic Cloud. -This integration includes predefined rules that automatically route AWS service logs to the respective integrations, which -include field mappings, ingest pipelines, predefined dashboards and ect. Here is a list of log types that are supported -by this integration: +Amazon Data Firehose integration offers users a way to stream logs and CloudWatch metrics from Firehose to Elastic Cloud. +This integration includes predefined rules that automatically route AWS service logs and CloudWatch metrics to the respective integrations, which +include field mappings, ingest pipelines, and predefined dashboards. + +Here is a list of log types that are supported by this integration: | AWS service log | Log destination | |--------------------|--------------------------| @@ -17,6 +18,31 @@ by this integration: | VPC Flow | Firehose, CloudWatch, S3 | | WAF | Firehose, CloudWatch. S3 | +Here is a list of CloudWatch metrics that are supported by this integration: + +| AWS service monitoring metrics | +|--------------------------------| +| API Gateway | +| DynamoDB | +| EBS | +| EC2 | +| ECS | +| ELB | +| EMR | +| Network Firewall | +| Kafka | +| Kinesis | +| Lambda | +| NATGateway | +| RDS | +| S3 | +| S3 Storage Lens | +| SNS | +| SQS | +| TransitGateway | +| Usage | +| VPN | + ## Limitation It is not possible to configure a delivery stream to send data to Elastic Cloud via PrivateLink (VPC endpoint). This is a current limitation in Firehose, which we are working with AWS to resolve. @@ -91,6 +117,12 @@ This is a current limitation in Firehose, which we are working with AWS to resol This parameter will increase the data volume in Elasticsearch and should be used with care. 3. Send data to the Firehose delivery stream - + 1. logs Consult the [AWS documentation](https://docs.aws.amazon.com/firehose/latest/dev/basic-write.html) for details on how to configure a variety of log sources to send data to Firehose delivery streams. + + 2. metrics + Consult the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup.html) + for details on how to set up a metric stream in CloudWatch and + [Custom setup with Firehose](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup-datalake.html) + to send metrics to Firehose. For Elastic, we only support JSON and OpenTelemetry 1.0.0 formats for the metrics. diff --git a/packages/awsfirehose/manifest.yml b/packages/awsfirehose/manifest.yml index 7be0e4f019a..c8439349ff6 100644 --- a/packages/awsfirehose/manifest.yml +++ b/packages/awsfirehose/manifest.yml @@ -1,15 +1,15 @@ -format_version: "3.0.0" +format_version: "3.1.0" name: awsfirehose title: Amazon Data Firehose -version: 1.0.0 -description: Stream logs from Amazon Data Firehose into Elastic Cloud. +version: 1.1.0 +description: Stream logs and metrics from Amazon Data Firehose into Elastic Cloud. type: integration categories: - observability - aws conditions: kibana: - version: "^8.10.1" + version: "^8.13.0" owner: github: elastic/obs-ds-hosted-services type: elastic