[O11y][Nginx] Rally benchmark nginx.error (#8762)

* rally benchmark nginx.error * remove extracted fields * revert changes * revert manifest * remove extracted fields * remove timezone config * remove date fields and hostname * update event.timezone
elastic · Jan 16, 2024 · bc6b4cd · bc6b4cd
1 parent 6a95ae7
commit bc6b4cd
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 0 deletions.
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark.yml
@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 nginx.error events ingested
+data_stream:
+  name: error
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./error-benchmark/template.ndjson
+    config:
+      path: ./error-benchmark/config.yml
+    fields:
+      path: ./error-benchmark/fields.yml
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark/config.yml
@@ -0,0 +1,19 @@
+fields:
+  - name: 'timestamp'
+    period: -24h # one day
+  - name: agent.id
+    value: "ef5e274d-4b53-45e6-943a-a5bcf1a6f523"
+  - name: log.level
+    enum: ["debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"]
+  - name: process.pid
+    range:
+      min: 1
+      max: 100000
+  - name: thread.id
+    range:
+      min: 1
+      max: 100000
+  - name: connection_id
+    range:
+      min: 1
+      max: 100000
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml b/packages/nginx/_dev/benchmark/rally/error-benchmark/fields.yml
@@ -0,0 +1,28 @@
+- name: agent.ephemeral_id
+  type: keyword
+- name: agent.ephemeral_id
+  type: keyword
+- name: agent.id
+  type: keyword
+- name: agent.name
+  type: keyword
+- name: agent.version
+  type: keyword
+- name: agent.version
+  type: keyword
+- name: connection_id
+  type: long
+- name: host.ip
+  type: ip
+- name: host.name
+  type: keyword
+- name: log.level
+  type: keyword
+- name: message
+  type: text
+- name: process.pid
+  type: long
+- name: thread.id
+  type: long
+- name: timestamp
+  type: date
diff --git a/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson b/packages/nginx/_dev/benchmark/rally/error-benchmark/template.ndjson
@@ -0,0 +1,72 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $agentId := generate "agent.id" }}
+{{- $agentVersion := generate "agent.version" }}
+{{- $agentName := generate "agent.name" }}
+{{- $agentEphemeralid := generate "agent.ephemeral_id" }}
+{{- $logLevel := generate "log.level" }}
+{{- $pid := generate "process.pid" }}
+{{- $threadId := generate "thread.id" }}
+{{- $connectionId := generate "connection_id" }}
+{
+    "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.000Z" }}",
+    "agent": {
+        "ephemeral_id": "{{ $agentEphemeralid }}",
+        "id": "{{ $agentId }}",
+        "name": "{{ $agentName }}",
+        "type": "filebeat",
+        "version": "8.8.0"
+    },
+    "data_stream": {
+        "dataset": "nginx.error",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.5.1"
+    },
+    "elastic_agent": {
+        "id": "{{ $agentEphemeralid }}",
+        "snapshot": false,
+        "version": "8.8.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "nginx.error",
+        "timezone": "{{ $timestamp.Format "-07:00" }}"
+    },
+     "host": {
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "docker-fleet-agent",
+        "id": "66392b0697b84641af8006d87aeb89f1",
+        "ip": [
+            "{{ generate "host.ip" }}"
+        ],
+        "mac": [
+            "02-42-AC-12-00-07"
+        ],
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.15.49-linuxkit",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.5 LTS (Focal Fossa)"
+        }
+    },
+    "input": {
+        "type": "log"
+    },
+    "log": {
+        "file": {
+            "path": "/var/log/nginx/error.log"
+        },
+        "offset": 0
+    },
+    "message": "{{$timestamp.Format "2006/01/02 15:04:05"}} [{{ $logLevel }}] {{ $pid }}#{{ $threadId }}: *{{ $connectionId }} {{generate "message"}}",
+    "tags": [
+        "nginx-error"
+    ]
+}