[O11y][SpringBoot] Add custom field for _id and update dashboard (#5935)

* Add custom field for _id and update dashboard * Update changelog * Add troubleshooting section in README * Address review comment and update README
elastic · Apr 28, 2023 · d707eca · d707eca
1 parent f8a7af3
commit d707eca
Show file tree

Hide file tree

Showing 22 changed files with 269 additions and 43 deletions.
diff --git a/packages/spring_boot/_dev/build/docs/README.md b/packages/spring_boot/_dev/build/docs/README.md
@@ -26,6 +26,99 @@ In order to ingest data from Spring Boot:
 </dependency>
 ```
 
+### Troubleshooting
+
+If **[Spring Boot] Audit Events panel** does not display older documents after upgrading to ``0.9.0`` or later versions, then this issue can be solved by reindexing the ``audit_events`` data stream's indices.
+
+To reindex the data, the following steps must be performed.
+
+1. Stop the data stream by going to `Integrations -> Spring Boot -> Integration policies` open the configuration of Spring Boot and disable the `Spring Boot Audit Events metrics` toggle to reindex ``audit_events`` data stream and save the integration.
+
+2. Copy data into the temporary index and delete the existing data stream and index template by performing the following steps in the Dev tools.
+
+```
+POST _reindex
+{
+  "source": {
+    "index": "<index_name>"
+  },
+  "dest": {
+    "index": "temp_index"
+  }
+}
+```
+Example:
+```
+POST _reindex
+{
+  "source": {
+    "index": "logs-spring_boot.audit_events-default"
+  },
+  "dest": {
+    "index": "temp_index"
+  }
+}
+```
+
+```
+DELETE /_data_stream/<data_stream>
+```
+Example:
+```
+DELETE /_data_stream/logs-spring_boot.audit_events-default
+```
+
+```
+DELETE _index_template/<index_template>
+```
+Example:
+```
+DELETE _index_template/logs-spring_boot.audit_events
+```
+3. Go to `Integrations ->  Spring Boot  -> Settings` and click on `Reinstall Spring Boot`.
+
+4. Copy data from temporary index to new index by performing the following steps in the Dev tools.
+
+```
+POST _reindex
+{
+  "source": {
+    "index": "temp_index"
+  },
+  "dest": {
+    "index": "<index_name>",
+    "op_type": "create"
+
+  }
+}
+```
+Example:
+```
+POST _reindex
+{
+  "source": {
+    "index": "temp_index"
+  },
+  "dest": {
+    "index": "logs-spring_boot.audit_events-default",
+    "op_type": "create"
+
+  }
+}
+```
+
+5. Verify data is reindexed completely.
+
+6. Start the data stream by going to the `Integrations -> Spring Boot -> Integration policies` and open configuration of integration and enable the `Spring Boot Audit Events metrics` toggle.
+
+7. Delete temporary index by performing the following step in the Dev tools.
+
+```
+DELETE temp_index
+```
+
+More details about reindexing can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html).
+
 ## Logs
 
 ### Audit Events logs

diff --git a/packages/spring_boot/changelog.yml b/packages/spring_boot/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.9.0"
+  changes:
+    - description: Add custom field for _id and update dashboard.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5935
 - version: "0.8.0"
   changes:
     - description: Migrate visualizations to lens.

diff --git a/...oot/data_stream/audit_events/_dev/test/pipeline/test-auditevent-metrics.log-expected.json b/...oot/data_stream/audit_events/_dev/test/pipeline/test-auditevent-metrics.log-expected.json
@@ -18,6 +18,7 @@
                         "remote_address": "192.168.32.6",
                         "session_id": "E175F2BFB6A22C9A60A116BE63D9F0C3"
                     },
+                    "document_id": "mb9+9ombaHnCqFCCunNAKb6CcrI=",
                     "principal": "anonymousUser",
                     "type": "AUTHENTICATION_SUCCESS"
                 }

diff --git a/packages/spring_boot/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/spring_boot/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml
@@ -52,11 +52,15 @@ processors:
       field: json.data.details.remoteAddress
       target_field: spring_boot.audit_events.data.remote_address
       ignore_missing: true
-      ignore_failure: true      
+      ignore_failure: true
+  - set:
+      field: spring_boot.audit_events.document_id
+      copy_from: _id
+      ignore_empty_value: true
   - remove:
       field:
-       - json
-       - input.type
+        - json
+        - input.type
       ignore_missing: true
       ignore_failure: true
   - script:

diff --git a/packages/spring_boot/data_stream/audit_events/fields/fields.yml b/packages/spring_boot/data_stream/audit_events/fields/fields.yml
@@ -13,6 +13,9 @@
             - name: session_id
               type: keyword
               description: Session ID of the Spring Boot application user.
+        - name: document_id
+          type: keyword
+          description: Unique document id generated by Elasticsearch.
         - name: principal
           type: keyword
           description: Restricts the events to those with the given principal.

diff --git a/packages/spring_boot/data_stream/audit_events/sample_event.json b/packages/spring_boot/data_stream/audit_events/sample_event.json
@@ -56,6 +56,7 @@
             "data": {
                 "remote_address": "192.168.144.2"
             },
+            "document_id": "Es32QTyIFsbGsH5nlZQxBDYnf18=",
             "principal": "actuator",
             "type": "AUTHENTICATION_SUCCESS"
         }

diff --git a/packages/spring_boot/docs/README.md b/packages/spring_boot/docs/README.md
@@ -26,6 +26,99 @@ In order to ingest data from Spring Boot:
 </dependency>
 ```
 
+### Troubleshooting
+
+If **[Spring Boot] Audit Events panel** does not display older documents after upgrading to ``0.9.0`` or later versions, then this issue can be solved by reindexing the ``audit_events`` data stream's indices.
+
+To reindex the data, the following steps must be performed.
+
+1. Stop the data stream by going to `Integrations -> Spring Boot -> Integration policies` open the configuration of Spring Boot and disable the `Spring Boot Audit Events metrics` toggle to reindex ``audit_events`` data stream and save the integration.
+
+2. Copy data into the temporary index and delete the existing data stream and index template by performing the following steps in the Dev tools.
+
+```
+POST _reindex
+{
+  "source": {
+    "index": "<index_name>"
+  },
+  "dest": {
+    "index": "temp_index"
+  }
+}
+```
+Example:
+```
+POST _reindex
+{
+  "source": {
+    "index": "logs-spring_boot.audit_events-default"
+  },
+  "dest": {
+    "index": "temp_index"
+  }
+}
+```
+
+```
+DELETE /_data_stream/<data_stream>
+```
+Example:
+```
+DELETE /_data_stream/logs-spring_boot.audit_events-default
+```
+
+```
+DELETE _index_template/<index_template>
+```
+Example:
+```
+DELETE _index_template/logs-spring_boot.audit_events
+```
+3. Go to `Integrations ->  Spring Boot  -> Settings` and click on `Reinstall Spring Boot`.
+
+4. Copy data from temporary index to new index by performing the following steps in the Dev tools.
+
+```
+POST _reindex
+{
+  "source": {
+    "index": "temp_index"
+  },
+  "dest": {
+    "index": "<index_name>",
+    "op_type": "create"
+
+  }
+}
+```
+Example:
+```
+POST _reindex
+{
+  "source": {
+    "index": "temp_index"
+  },
+  "dest": {
+    "index": "logs-spring_boot.audit_events-default",
+    "op_type": "create"
+
+  }
+}
+```
+
+5. Verify data is reindexed completely.
+
+6. Start the data stream by going to the `Integrations -> Spring Boot -> Integration policies` and open configuration of integration and enable the `Spring Boot Audit Events metrics` toggle.
+
+7. Delete temporary index by performing the following step in the Dev tools.
+
+```
+DELETE temp_index
+```
+
+More details about reindexing can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html).
+
 ## Logs
 
 ### Audit Events logs
@@ -95,6 +188,7 @@ An example event for `audit_events` looks as following:
             "data": {
                 "remote_address": "192.168.144.2"
             },
+            "document_id": "Es32QTyIFsbGsH5nlZQxBDYnf18=",
             "principal": "actuator",
             "type": "AUTHENTICATION_SUCCESS"
         }
@@ -120,6 +214,7 @@ An example event for `audit_events` looks as following:
 | event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword |
 | spring_boot.audit_events.data.remote_address | Remote Address of the Spring Boot application user. | keyword |
 | spring_boot.audit_events.data.session_id | Session ID of the Spring Boot application user. | keyword |
+| spring_boot.audit_events.document_id | Unique document id generated by Elasticsearch. | keyword |
 | spring_boot.audit_events.principal | Restricts the events to those with the given principal. | keyword |
 | spring_boot.audit_events.type | Authentication type. | keyword |
 | tags | List of keywords used to tag each event. | keyword |

diff --git a/packages/spring_boot/kibana/dashboard/spring_boot-6fe11330-c7f1-11ec-986a-f1e84447fd00.json b/packages/spring_boot/kibana/dashboard/spring_boot-6fe11330-c7f1-11ec-986a-f1e84447fd00.json
@@ -56,10 +56,10 @@
         "title": "[Spring Boot] Audit Events",
         "version": 1
     },
-    "coreMigrationVersion": "8.1.0",
+    "coreMigrationVersion": "8.2.0",
     "id": "spring_boot-6fe11330-c7f1-11ec-986a-f1e84447fd00",
     "migrationVersion": {
-        "dashboard": "8.1.0"
+        "dashboard": "8.2.0"
     },
     "references": [
         {

diff --git a/packages/spring_boot/kibana/dashboard/spring_boot-6fe782d0-ca15-11ec-b4b1-b57f75afa7dd.json b/packages/spring_boot/kibana/dashboard/spring_boot-6fe782d0-ca15-11ec-b4b1-b57f75afa7dd.json
@@ -57,10 +57,10 @@
         "title": "[Spring Boot] Access logs",
         "version": 1
     },
-    "coreMigrationVersion": "8.1.0",
+    "coreMigrationVersion": "8.2.0",
     "id": "spring_boot-6fe782d0-ca15-11ec-b4b1-b57f75afa7dd",
     "migrationVersion": {
-        "dashboard": "8.1.0"
+        "dashboard": "8.2.0"
     },
     "references": [
         {

diff --git a/packages/spring_boot/kibana/dashboard/spring_boot-c89e1510-ca15-11ec-b4b1-b57f75afa7dd.json b/packages/spring_boot/kibana/dashboard/spring_boot-c89e1510-ca15-11ec-b4b1-b57f75afa7dd.json
@@ -108,10 +108,10 @@
         "title": "[Spring Boot] Memory and Threads",
         "version": 1
     },
-    "coreMigrationVersion": "8.1.0",
+    "coreMigrationVersion": "8.2.0",
     "id": "spring_boot-c89e1510-ca15-11ec-b4b1-b57f75afa7dd",
     "migrationVersion": {
-        "dashboard": "8.1.0"
+        "dashboard": "8.2.0"
     },
     "references": [
         {

diff --git a/packages/spring_boot/kibana/dashboard/spring_boot-e8507040-c7b0-11ec-9c82-4b3dfe36a746.json b/packages/spring_boot/kibana/dashboard/spring_boot-e8507040-c7b0-11ec-9c82-4b3dfe36a746.json
@@ -142,7 +142,8 @@
                         "visualizationType": "lnsMetric"
                     },
                     "enhancements": {},
-                    "hidePanelTitles": false
+                    "hidePanelTitles": false,
+                    "type": "lens"
                 },
                 "gridData": {
                     "h": 6,
@@ -154,7 +155,7 @@
                 "panelIndex": "8da90917-936b-49b2-a845-24db7da4681e",
                 "title": "Total successful requests [Logs Spring Boot]",
                 "type": "lens",
-                "version": "8.1.0"
+                "version": "8.2.0"
             },
             {
                 "embeddableConfig": {
@@ -280,7 +281,8 @@
                         "visualizationType": "lnsMetric"
                     },
                     "enhancements": {},
-                    "hidePanelTitles": false
+                    "hidePanelTitles": false,
+                    "type": "lens"
                 },
                 "gridData": {
                     "h": 6,
@@ -292,7 +294,7 @@
                 "panelIndex": "44a2b9a5-626c-426b-8bbb-921b56040444",
                 "title": "Total failed requests [Logs Spring Boot]",
                 "type": "lens",
-                "version": "8.1.0"
+                "version": "8.2.0"
             },
             {
                 "embeddableConfig": {
@@ -345,6 +347,7 @@
                                                                 "decimals": 1
                                                             }
                                                         },
+                                                        "showArrayValues": true,
                                                         "sortField": "@timestamp"
                                                     },
                                                     "scale": "ratio",
@@ -414,7 +417,8 @@
                         "visualizationType": "lnsMetric"
                     },
                     "enhancements": {},
-                    "hidePanelTitles": false
+                    "hidePanelTitles": false,
+                    "type": "lens"
                 },
                 "gridData": {
                     "h": 6,
@@ -426,7 +430,7 @@
                 "panelIndex": "d227dc3c-7902-44e7-b367-36bee05b96ab",
                 "title": "Total memory of current thread in system [Metrics Spring Boot]",
                 "type": "lens",
-                "version": "8.1.0"
+                "version": "8.2.0"
             },
             {
                 "embeddableConfig": {
@@ -499,10 +503,10 @@
         "title": "[Spring Boot] Overview Dashboard",
         "version": 1
     },
-    "coreMigrationVersion": "8.1.0",
+    "coreMigrationVersion": "8.2.0",
     "id": "spring_boot-e8507040-c7b0-11ec-9c82-4b3dfe36a746",
     "migrationVersion": {
-        "dashboard": "8.1.0"
+        "dashboard": "8.2.0"
     },
     "references": [
         {