Rally benchmark aws.sqs (#8429)

* bump github.com/elastic/package-spec/[email protected] * add aws.sqs rally benchmark * recude to 60m timestamp period
elastic · Nov 16, 2023 · 3087d39 · 3087d39
1 parent e974a50
commit 3087d39
Show file tree

Hide file tree

Showing 4 changed files with 145 additions and 0 deletions.
diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml b/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml
@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 aws.sqs events ingested
+data_stream:
+  name: sqs
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./sqs-benchmark/template.ndjson
+    config:
+      path: ./sqs-benchmark/config.yml
+    fields:
+      path: ./sqs-benchmark/fields.yml
diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/sqs-benchmark/config.yml
@@ -0,0 +1,59 @@
+fields:
+  - name: timestamp
+    period: 60m # one hour
+  - name: Region
+    enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"]
+    cardinality: 25
+  - name: Visible
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: Deleted
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: NotVisible
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: Delayed
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: Received
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: Sent
+    range:
+      min: 0
+      max: 1000
+    fuzziness: 0.05
+  - name: EmptyReceives
+    range:
+      min: 0
+      max: 10
+    fuzziness: 0.1
+  - name: SentMessageSize
+    range:
+      min: 1024
+      max: 262144
+    fuzziness: 0.5
+  - name: OldestMessageAge
+    range:
+      min: 1
+      max: 7200
+    fuzziness: 0.05
+  - name: QueueName
+    cardinality: 200
+  - name: EventDuration
+    range:
+      min: 1
+      max: 1000
+  - name: TagsCreatedBy
+    cardinality: 200
diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark/fields.yml b/packages/aws/_dev/benchmark/rally/sqs-benchmark/fields.yml
@@ -0,0 +1,30 @@
+- name: timestamp
+  type: date
+- name: Region
+  type: keyword
+- name: Visible
+  type: double
+- name: Deleted
+  type: double
+- name: NotVisible
+  type: double
+- name: Delayed
+  type: double
+- name: Received
+  type: double
+- name: Sent
+  type: double
+- name: EmptyReceives
+  type: double
+- name: SentMessageSize
+  type: double
+- name: OldestMessageAge
+  type: double
+- name: QueueName
+  type: keyword
+  example: a-queue-name
+- name: EventDuration
+  type: long
+- name: TagsCreatedBy
+  type: keyword
+  example: First Last
diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark/template.ndjson b/packages/aws/_dev/benchmark/rally/sqs-benchmark/template.ndjson
@@ -0,0 +1,42 @@
+{{- $queueName := generate "QueueName" -}}
+{{- $timestamp := generate "timestamp" -}}
+{
+    "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}",
+    "agent": {
+        "name": "docker-fleet-agent",
+        "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864",
+        "type": "metricbeat",
+        "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15",
+        "version": "8.3.2"
+    },
+    "elastic_agent": { "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "version": "8.3.2", "snapshot": false },
+    "cloud": { "provider": "aws", "region": "{{ generate "Region" }}", "account": { "name": "elastic-beats", "id": "000000000000" } },
+    "ecs": { "version": "8.0.0" },
+    "service": { "type": "aws" },
+    "data_stream": { "namespace": "ep", "type": "metrics", "dataset": "aws.sqs" },
+    "metricset": { "period": 300, "name": "cloudwatch" },
+    "event": { "duration": {{ generate "EventDuration" }},
+        "agent_id_status": "verified",
+        "module": "aws",
+        "dataset": "aws.sqs"
+    },
+    "aws": {
+        "cloudwatch": { "namespace": "AWS/SQS" },
+        "dimensions": { "QueueName": "{{ $queueName }}" },
+        "sqs": {
+            "queue": { "name": "{{ $queueName }}" },
+            "metrics": {
+                "ApproximateAgeOfOldestMessage": { "avg": {{ generate "OldestMessageAge" }} },
+                "ApproximateNumberOfMessagesDelayed": { "avg": {{ generate "Delayed" }} },
+                "ApproximateNumberOfMessagesNotVisible": { "avg": {{ generate "NotVisible" }} },
+                "ApproximateNumberOfMessagesVisible": { "avg": {{ generate "Visible" }} },
+                "NumberOfMessagesDeleted": { "avg": {{ generate "Deleted" }} },
+                "NumberOfMessagesReceived": { "avg": {{ generate "Received" }} },
+                "NumberOfMessagesSent": { "avg": {{ generate "Sent" }} },
+                "NumberOfEmptyReceives": { "avg": {{ generate "EmptyReceives" }} },
+                "SentMessageSize": { "avg": {{ generate "SentMessageSize" }} }
+            }
+        },
+        "tags": { "createdBy": "{{ generate "TagsCreatedBy" }}" }
+    }
+}