modsecurity: relax timestamp parsing and improve error handling (elas…

…tic#5418)
agithomas · Mar 20, 2023 · c1234e8 · c1234e8
1 parent cbe0ca8
commit c1234e8
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 4 deletions.
diff --git a/packages/modsecurity/changelog.yml b/packages/modsecurity/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Handle sub-second resolution timestamps.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5418
+    - description: Improve error handling.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5418
 - version: "1.5.1"
   changes:
     - description: Add support for time zone configuration.

diff --git a/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-apache-audit.log b/packages/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-apache-audit.log
diff --git a/...s/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-apache-audit.log-expected.json b/...s/modsecurity/data_stream/auditlog/_dev/test/pipeline/test-apache-audit.log-expected.json
diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml
@@ -45,20 +45,31 @@ processors:
       formats:
         - d/MMM/yyyy:HH:mm:ss
         - d/MMM/yyyy:HH:mm:ss Z
+        - d/MMM/yyyy:HH:mm:ss.SSS
+        - d/MMM/yyyy:HH:mm:ss.SSS Z
+        - d/MMM/yyyy:HH:mm:ss.SSSSSS
+        - d/MMM/yyyy:HH:mm:ss.SSSSSS Z
       on_failure:
         # Try to re-parse as UTC to catch when TZ is invalid or unknown.
         - remove:
             field: event.timezone
             ignore_missing: true
         - date:
             field: _temps.date
+            tag: "time_date"
             formats:
               - d/MMM/yyyy:HH:mm:ss
               - d/MMM/yyyy:HH:mm:ss Z
+              - d/MMM/yyyy:HH:mm:ss.SSS
+              - d/MMM/yyyy:HH:mm:ss.SSS Z
+              - d/MMM/yyyy:HH:mm:ss.SSSSSS
+              - d/MMM/yyyy:HH:mm:ss.SSSSSS Z
             on_failure:
               - append:
                   field: error.message
-                  value: "{{{ _ingest.on_failure_message }}}"
+                  value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
+              - fail:
+                  message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
 
   # rename ecs
   - rename:

diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml
@@ -32,5 +32,8 @@ processors:
 
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
       value: "{{ _ingest.on_failure_message }}"
diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml
@@ -35,20 +35,31 @@ processors:
       formats:
         - E MMM dd HH:mm:ss yyyy
         - E MMM  d HH:mm:ss yyyy
+        - d/MMM/yyyy:HH:mm:ss.SSS
+        - d/MMM/yyyy:HH:mm:ss.SSS Z
+        - d/MMM/yyyy:HH:mm:ss.SSSSSS
+        - d/MMM/yyyy:HH:mm:ss.SSSSSS Z
       on_failure:
         # Try to re-parse as UTC to catch when TZ is invalid or unknown.
         - remove:
             field: event.timezone
             ignore_missing: true
         - date:
             field: _temps.date
+            tag: "time_date"
             formats:
               - E MMM dd HH:mm:ss yyyy
               - E MMM  d HH:mm:ss yyyy
+              - d/MMM/yyyy:HH:mm:ss.SSS
+              - d/MMM/yyyy:HH:mm:ss.SSS Z
+              - d/MMM/yyyy:HH:mm:ss.SSSSSS
+              - d/MMM/yyyy:HH:mm:ss.SSSSSS Z
             on_failure:
               - append:
                   field: error.message
-                  value: "{{{ _ingest.on_failure_message }}}"
+                  value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
+              - fail:
+                  message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
 
 # rename ecs
   - rename:

diff --git a/packages/modsecurity/manifest.yml b/packages/modsecurity/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: modsecurity
 title: "ModSecurity Audit"
-version: "1.5.1"
+version: "1.6.0"
 license: basic
 description: Collect logs from ModSecurity with Elastic Agent
 type: integration