[O11y][AWS] Rally benchmark aws.vpcflow

packages/aws/_dev/benchmark/rally/vpcflow-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 aws.vpcflow events ingested
+data_stream:
+  name: vpcflow
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./vpcflow-benchmark/template.ndjson
+    config:
+      path: ./vpcflow-benchmark/config.yml
+    fields:
+      path: ./vpcflow-benchmark/fields.yml

packages/aws/_dev/benchmark/rally/vpcflow-benchmark/config.yml

@@ -0,0 +1,59 @@
+fields:
+  - name: timestamp
+    period: -24h
+  - name: cloud_region
+    enum: ["ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"]
+  - name: network_direction
+    enum: ["ingress", "egress"]
+  - name: duration_start
+    enum: ["-1h", "-2h", "-3h", "-4h"]
+  - name: duration_end
+    enum: ["-30m", "-20m", "-10m", "-40m"]
+  - name: long_num
+    range:
+      min: 100000000
+      max: 200000000
+  - name: aws_vpcflow_version
+    enum: ["2", "3", "5"]
+  - name: aws_vpcflow_account_id
+    range:
+      min: 100000000000
+      max: 300000000000
+  - name: aws_vpcflow_action
+    enum: ["ACCEPT", "REJECT"]
+  - name: log_status
+    enum: ["OK", "NODATA", "SKIPDATA"]
+  - name: aws_vpcflow_tcp_flags
+    enum: ["0", "1", "2", "3", "4", "18", "19"]
+  - name: aws_vpcflow_pkt_dst_service
+    enum: ["AMAZON", "AMAZON_APPFLOW", "AMAZON_CONNECT", "API_GATEWAY", "CHIME_MEETINGS", "CHIME_VOICECONNECTOR", "CLOUD9", "CLOUDFRONT", "CODEBUILD", "DYNAMODB", "EBS", "EC2", "EC2_INSTANCE_CONNECT", "GLOBALACCELERATOR", "KINESIS_VIDEO_STREAMS", "ROUTE53", "ROUTE53_HEALTHCHECKS", "ROUTE53_HEALTHCHECKS_PUBLISHING", "ROUTE53_RESOLVER", "S3", "WORKSPACES_GATEWAYS"]
+  - name: aws_vpcflow_pkt_src_service
+    enum: ["AMAZON", "AMAZON_APPFLOW", "AMAZON_CONNECT", "API_GATEWAY", "CHIME_MEETINGS", "CHIME_VOICECONNECTOR", "CLOUD9", "CLOUDFRONT", "CODEBUILD", "DYNAMODB", "EBS", "EC2", "EC2_INSTANCE_CONNECT", "GLOBALACCELERATOR", "KINESIS_VIDEO_STREAMS", "ROUTE53", "ROUTE53_HEALTHCHECKS", "ROUTE53_HEALTHCHECKS_PUBLISHING", "ROUTE53_RESOLVER", "S3", "WORKSPACES_GATEWAYS"]
+  - name: aws_vpcflow_traffic_path
+    range:
+      min: 1
+      max: 8
+  - name: aws_vpcflow_sublocation_type
+    enum: ["wavelength", "outpost", "localzone"]
+  - name: aws_vpcflow_srcport
+    range:
+      min: 0
+      max: 65535
+  - name: aws_vpcflow_dstport
+    range:
+      min: 0
+      max: 65535
+  - name: aws_vpcflow_protocol
+    enum: ["1", "2", "6", "17", "47", "58", "132"]
+  - name: aws_vpcflow_packets
+    range:
+      min: 0
+      max: 100
+  - name: aws_vpcflow_bytes
+    range:
+      min: 0
+      max: 10000
+  - name: bucket_num
+    range:
+      min: 63461
+      max: 63471

packages/aws/_dev/benchmark/rally/vpcflow-benchmark/fields.yml

@@ -0,0 +1,61 @@
+- name: timestamp
+  type: date
+- name: cloud_region
+  type: keyword
+- name: network_direction
+  type: keyword
+- name: aws_vpcflow_version
+  type: keyword
+- name: aws_vpcflow_account_id
+  type: long
+- name: aws_vpcflow_interface_id
+  type: keyword
+- name: aws_vpcflow_action
+  type: keyword
+- name: log_status
+  type: keyword
+- name: aws_vpcflow_pkt_srcaddr
+  type: ip
+- name: aws_vpcflow_pkt_dstaddr
+  type: ip
+- name: aws_vpcflow_vpc_id
+  type: keyword
+- name: long_num
+  type: long
+- name: aws_vpcflow_subnet_id
+  type: keyword
+- name: aws_vpcflow_tcp_flags
+  type: keyword
+- name: aws_vpcflow_pkt_dst_service
+  type: keyword
+- name: aws_vpcflow_pkt_src_service
+  type: keyword
+- name: aws_vpcflow_traffic_path
+  type: long
+- name: aws_vpcflow_sublocation_type
+  type: keyword
+- name: aws_vpcflow_sublocation_id
+  type: keyword
+- name: aws_vpcflow_srcaddr
+  type: ip
+- name: aws_vpcflow_dstaddr
+  type: ip
+- name: aws_vpcflow_srcport
+  type: long
+- name: aws_vpcflow_dstport
+  type: long
+- name: aws_vpcflow_protocol
+  type: keyword
+- name: aws_vpcflow_packets
+  type: long
+- name: aws_vpcflow_bytes
+  type: long
+- name: duration_start
+  type: keyword
+- name: duration_end
+  type: keyword
+- name: file_name
+  type: keyword
+  example: extra-samples
+- name: bucket_num
+  type: long

packages/aws/_dev/benchmark/rally/vpcflow-benchmark/template.ndjson

@@ -0,0 +1,86 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $cloud_region := generate "cloud_region" }}
+{{- $cloud_availability_zone := awsAZFromRegion (print $cloud_region) }}
+{{- $network_direction := generate "network_direction" }}
+{{- $duration_start := generate "duration_start" }}
+{{- $duration_end := generate "duration_end" }}
+{{- $aws_vpcflow_start := $timestamp | date_modify (print $duration_start) }}
+{{- $aws_vpcflow_end := $timestamp | date_modify (print $duration_end) }}
+{{- $aws_vpcflow_version := generate "aws_vpcflow_version" }}
+{{- $aws_vpcflow_account_id := generate "aws_vpcflow_account_id" }}
+{{- $aws_vpcflow_interface_id := generate "aws_vpcflow_interface_id" }}
+{{- $aws_vpcflow_action := generate "aws_vpcflow_action" }}
+{{- $log_status := generate "log_status" }}
+{{- $aws_vpcflow_pkt_srcaddr := generate "aws_vpcflow_pkt_srcaddr" }}
+{{- $aws_vpcflow_pkt_dstaddr := generate "aws_vpcflow_pkt_dstaddr" }}
+{{- $aws_vpcflow_vpc_id := generate "aws_vpcflow_vpc_id" }}
+{{- $aws_vpcflow_subnet_id := generate "aws_vpcflow_subnet_id" }}
+{{- $aws_vpcflow_tcp_flags := generate "aws_vpcflow_tcp_flags" }}
+{{- $aws_vpcflow_pkt_dst_service := generate "aws_vpcflow_pkt_dst_service" }}
+{{- $aws_vpcflow_pkt_src_service := generate "aws_vpcflow_pkt_src_service" }}
+{{- $aws_vpcflow_traffic_path := generate "aws_vpcflow_traffic_path" }}
+{{- $aws_vpcflow_sublocation_type := generate "aws_vpcflow_sublocation_type" }}
+{{- $aws_vpcflow_sublocation_id := generate "aws_vpcflow_sublocation_id" }}
+{{- $aws_vpcflow_srcaddr := generate "aws_vpcflow_srcaddr" }}
+{{- $aws_vpcflow_dstaddr := generate "aws_vpcflow_dstaddr" }}
+{{- $aws_vpcflow_srcport := generate "aws_vpcflow_srcport" }}
+{{- $aws_vpcflow_dstport := generate "aws_vpcflow_dstport" }}
+{{- $aws_vpcflow_protocol := generate "aws_vpcflow_protocol" }}
+{{- $aws_vpcflow_packets := generate "aws_vpcflow_packets" }}
+{{- $aws_vpcflow_bytes := generate "aws_vpcflow_bytes" }}
+{{- $long_num := generate "long_num" }}
+{{- $file_name := generate "file_name" }}
+{{- $bucket_num := generate "bucket_num" }}
+{
+    "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.000Z" }}",
+    "agent": {
+        "ephemeral_id": "22ed892c-43bd-408a-9121-65e2f5b6a56e",
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "name": "aws-scale-123456",
+        "type": "filebeat",
+        "version": "8.8.0"
+    },
+    "aws": {
+        "s3": {
+            "bucket": {
+                "arn": "arn:aws:s3:::elastic-package-aws-bucket-{{ $bucket_num }}",
+                "name": "elastic-package-aws-bucket-{{ $bucket_num }}"
+            },
+            "object": {
+                "key": "{{ $file_name }}.log"
+            }
+        }
+    },
+    "data_stream": {
+        "dataset": "aws.vpcflow",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "elastic_agent": {
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "snapshot": false,
+        "version": "8.8.0"
+    },
+    "event": {
+        "dataset": "aws.vpcflow",
+        {{- if eq $log_status "OK"}}
+        "original" : "{\"message\":\"{{ $aws_vpcflow_version }} {{ $aws_vpcflow_account_id }} eni-{{ $aws_vpcflow_interface_id }}{{ mul $long_num 2 }} {{ $aws_vpcflow_srcaddr }} {{ $aws_vpcflow_dstaddr }} {{ $aws_vpcflow_srcport }} {{ $aws_vpcflow_dstport }} {{ $aws_vpcflow_protocol }} {{ $aws_vpcflow_packets }} {{ $aws_vpcflow_bytes }} {{ $aws_vpcflow_start.Unix }} {{ $aws_vpcflow_end.Unix }} {{ $aws_vpcflow_action }} {{ $log_status }} vpc-{{ $aws_vpcflow_vpc_id }}{{ $long_num }} subnet-{{ $aws_vpcflow_subnet_id }}{{ $long_num }} i-{{ $long_num }}{{ $long_num }} {{ $aws_vpcflow_tcp_flags }} IPv4 {{ $aws_vpcflow_pkt_srcaddr }} {{ $aws_vpcflow_pkt_dstaddr }} {{ $cloud_region }} {{ $cloud_availability_zone }} {{ $aws_vpcflow_sublocation_type }} {{ $aws_vpcflow_sublocation_id }} {{ $aws_vpcflow_pkt_src_service }} {{ $aws_vpcflow_pkt_dst_service }} {{ $network_direction }} {{ $aws_vpcflow_traffic_path }}\"}"  
+        {{- else}}
+        "original" : "{\"message\":\"{{ $aws_vpcflow_version }} {{ $aws_vpcflow_account_id }} eni-{{ $aws_vpcflow_interface_id }}{{ mul $long_num 2 }} - - - - - - - {{ $aws_vpcflow_start.Unix }} {{ $aws_vpcflow_end.Unix }} - {{ $log_status }}\"}"     
+        {{- end}}
+    },
+    "input": {
+        "type": "aws-s3"
+    },
+    "log": {
+        "file": {
+            "path": "https://elastic-package-aws-bucket-{{ $bucket_num }}.s3.{{ $cloud_region }}.amazonaws.com/{{ $file_name }}.log"
+        },
+        "offset": 338
+    },
+    "tags": [
+        "preserve_original_event",
+        "forwarded",
+        "aws-vpcflow"
+    ]
+}