From bc6b4cd6f49b82281c801fb6ada9c55a5462477e Mon Sep 17 00:00:00 2001
From: Aliabbas Attarwala <124054599+aliabbas-elastic@users.noreply.github.com>
Date: Tue, 16 Jan 2024 18:09:38 +0530
Subject: [PATCH] [O11y][Nginx] Rally benchmark `nginx.error` (#8762)

* rally benchmark nginx.error

* remove extracted fields

* revert changes

* revert manifest

* remove extracted fields

* remove timezone config

* remove date fields and hostname

* update event.timezone
---
 .../_dev/benchmark/rally/error-benchmark.yml  | 14 ++++
 .../rally/error-benchmark/config.yml          | 19 +++++
 .../rally/error-benchmark/fields.yml          | 28 ++++++++
 .../rally/error-benchmark/template.ndjson     | 72 +++++++++++++++++++
 4 files changed, 133 insertions(+)
 create mode 100644 packages/nginx/_dev/benchmark/rally/error-benchmark.yml
 create mode 100644 packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml
 create mode 100644 packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml
 create mode 100644 packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson

diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark.yml
new file mode 100644
index 00000000000..0710efbe796
--- /dev/null
+++ b/packages/nginx/_dev/benchmark/rally/error-benchmark.yml
@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 nginx.error events ingested
+data_stream:
+  name: error
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./error-benchmark/template.ndjson
+    config:
+      path: ./error-benchmark/config.yml
+    fields:
+      path: ./error-benchmark/fields.yml
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml
new file mode 100644
index 00000000000..1ccf19deb89
--- /dev/null
+++ b/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml
@@ -0,0 +1,19 @@
+fields:
+  - name: 'timestamp'
+    period: -24h # one day
+  - name: agent.id
+    value: "ef5e274d-4b53-45e6-943a-a5bcf1a6f523"
+  - name: log.level
+    enum: ["debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"]
+  - name: process.pid
+    range:
+      min: 1
+      max: 100000
+  - name: thread.id
+    range:
+      min: 1
+      max: 100000
+  - name: connection_id
+    range:
+      min: 1
+      max: 100000
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml
new file mode 100644
index 00000000000..990f7d91ace
--- /dev/null
+++ b/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml
@@ -0,0 +1,28 @@
+- name: agent.ephemeral_id
+  type: keyword
+- name: agent.ephemeral_id
+  type: keyword
+- name: agent.id
+  type: keyword
+- name: agent.name
+  type: keyword
+- name: agent.version
+  type: keyword
+- name: agent.version
+  type: keyword
+- name: connection_id
+  type: long
+- name: host.ip
+  type: ip
+- name: host.name
+  type: keyword
+- name: log.level
+  type: keyword
+- name: message
+  type: text
+- name: process.pid
+  type: long
+- name: thread.id
+  type: long
+- name: timestamp
+  type: date
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson b/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson
new file mode 100644
index 00000000000..b8e9abf98fa
--- /dev/null
+++ b/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson
@@ -0,0 +1,72 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $agentId := generate "agent.id" }}
+{{- $agentVersion := generate "agent.version" }}
+{{- $agentName := generate "agent.name" }}
+{{- $agentEphemeralid := generate "agent.ephemeral_id" }}
+{{- $logLevel := generate "log.level" }}
+{{- $pid := generate "process.pid" }}
+{{- $threadId := generate "thread.id" }}
+{{- $connectionId := generate "connection_id" }}
+{
+    "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.000Z" }}",
+    "agent": {
+        "ephemeral_id": "{{ $agentEphemeralid }}",
+        "id": "{{ $agentId }}",
+        "name": "{{ $agentName }}",
+        "type": "filebeat",
+        "version": "8.8.0"
+    },
+    "data_stream": {
+        "dataset": "nginx.error",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.5.1"
+    },
+    "elastic_agent": {
+        "id": "{{ $agentEphemeralid }}",
+        "snapshot": false,
+        "version": "8.8.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "nginx.error",
+        "timezone": "{{ $timestamp.Format "-07:00" }}"
+    },
+     "host": {
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "docker-fleet-agent",
+        "id": "66392b0697b84641af8006d87aeb89f1",
+        "ip": [
+            "{{ generate "host.ip" }}"
+        ],
+        "mac": [
+            "02-42-AC-12-00-07"
+        ],
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.15.49-linuxkit",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.5 LTS (Focal Fossa)"
+        }
+    },
+    "input": {
+        "type": "log"
+    },
+    "log": {
+        "file": {
+            "path": "/var/log/nginx/error.log"
+        },
+        "offset": 0
+    },
+    "message": "{{$timestamp.Format "2006/01/02 15:04:05"}} [{{ $logLevel }}] {{ $pid }}#{{ $threadId }}: *{{ $connectionId }} {{generate "message"}}",
+    "tags": [
+        "nginx-error"
+    ]
+}
\ No newline at end of file