Skip to content

Commit

Permalink
Remove unused fields from the vulnerabilities index template (#369)
Browse files Browse the repository at this point in the history
* Remove unused fields from the vulnerabilities index template

* Fix and use remove_multi_fields function
  • Loading branch information
AlexRuiz7 authored Sep 4, 2024
1 parent 379fc66 commit e7f83c3
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 41 deletions.
28 changes: 27 additions & 1 deletion ecs/generate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,19 @@ show_usage() {
echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
}

# Function to remove multi-fields from the generated index template
remove_multi_fields() {
local IN_FILE="$1"
local OUT_FILE="$2"

jq 'del(
.mappings.properties.host.properties.os.properties.full.fields,
.mappings.properties.host.properties.os.properties.name.fields,
.mappings.properties.vulnerability.properties.description.fields
)' "$IN_FILE" > "$OUT_FILE"
}


# Function to generate mappings
generate_mappings() {
local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
Expand All @@ -34,8 +47,21 @@ generate_mappings() {
echo "Replacing \"match_only_text\" type with \"text\""
find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;

local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json"
local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json"

# Delete the "tags" field from the index template
echo "Deleting the \"tags\" field from the index template"
jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE"
mv "$OUT_FILE" "$IN_FILE"

# Remove multi-fields from the generated index template
echo "Removing multi-fields from the index template"
remove_multi_fields "$IN_FILE" "$OUT_FILE"
mv "$OUT_FILE" "$IN_FILE"

# Transform legacy index template for OpenSearch compatibility
cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
cat "$IN_FILE" | jq '{
"index_patterns": .index_patterns,
"priority": .order,
"template": {
Expand Down
36 changes: 11 additions & 25 deletions ecs/vulnerability-detector/event-generator/event_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@

# This script generates sample events and injects them into the Wazuh Indexer.
# The events follow the Elastic Common Schema (ECS) format, and contains the following fields:
# - ecs
# - base
# - event
# - agent
# - package
# - host
# - vulnerability
# - wazuh (custom)
#
# This is an ad-hoc script for the vulnearbility module. Extend to support other modules.
# This is an ad-hoc script for the vulnerability module. Extend to support other modules.

import datetime
import random
Expand Down Expand Up @@ -102,7 +100,6 @@ def generate_random_host():
version = f'{random.randint(0, 99)}.{random.randint(0, 99)}'
host = {
'os': {
'family': family,
'full': f'{family} {version}',
'kernel': f'{version}kernel{random.randint(0, 99)}',
'name': f'{family} {version}',
Expand Down Expand Up @@ -141,11 +138,6 @@ def generate_random_package():
return package


def generate_random_tags():
tags = [f'tag{random.randint(0, 99)}' for _ in range(random.randint(0, 9))]
return tags


def generate_random_vulnerability():
id = random.randint(0, 9999)
vulnerability = {
Expand Down Expand Up @@ -176,9 +168,9 @@ def generate_random_wazuh():
'name': f'wazuh-cluster-{random.randint(0,10)}',
'node': f'wazuh-cluster-node-{random.randint(0,10)}'
},
'manager': {
'name': f'wazuh-manager-{random.randint(0,10)}'
},
# 'manager': {
# 'name': f'wazuh-manager-{random.randint(0,10)}'
# },
'schema': {
'version': '1.7.0'
},
Expand All @@ -190,15 +182,9 @@ def generate_random_data(number):
data = []
for _ in range(number):
event_data = {
# '@timestamp': generate_random_date(),
'agent': generate_random_agent(),
# 'ecs': {'version': '1.7.0'},
# 'event': generate_random_event(),
'host': generate_random_host(),
# 'labels': generate_random_labels(),
'message': f'message{random.randint(0, 99999)}',
'package': generate_random_package(),
'tags': generate_random_tags(),
'vulnerability': generate_random_vulnerability(),
'wazuh': generate_random_wazuh()
}
Expand Down Expand Up @@ -227,7 +213,7 @@ def inject_events(ip, port, index, username, password, data):

def main():
try:
number = int(input("How many events do you want to generate? "))
number = int(input("How many events do you want to generate? ").strip() or 50)
except ValueError:
logging.error("Invalid input. Please enter a valid number.")
return
Expand All @@ -245,11 +231,11 @@ def main():
inject = input(
"Do you want to inject the generated data into your indexer? (y/n) ").strip().lower()
if inject == 'y':
ip = input("Enter the IP of your Indexer: ")
port = input("Enter the port of your Indexer: ")
index = input("Enter the index name: ")
username = input("Username: ")
password = input("Password: ")
ip = input("Enter the IP of your Indexer: ").strip() or "localhost"
port = input("Enter the port of your Indexer: ").strip() or 9200
index = input("Enter the index name: ").strip() or "wazuh-states-vulnerability-test"
username = input("Username: ").strip() or "admin"
password = input("Password: ").strip()
inject_events(ip, port, index, username, password, data)


Expand Down
5 changes: 0 additions & 5 deletions ecs/vulnerability-detector/fields/custom/wazuh.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
level: custom
description: >
Wazuh cluster node name.
- name: manager.name
type: keyword
level: custom
description: >
Wazuh manager name. Used by dashboards to filter results on single node deployments.
- name: schema.version
type: keyword
level: custom
Expand Down
9 changes: 7 additions & 2 deletions ecs/vulnerability-detector/fields/subset.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,20 @@ fields:
base:
fields:
tags: []
message: ""
agent:
fields: "*"
package:
fields: "*"
host:
fields:
os:
fields: "*"
fields:
full: ""
kernel: ""
name: ""
platform: ""
type: ""
version: ""
vulnerability:
fields: "*"
wazuh:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"index_patterns": [
"wazuh-states-vulnerabilities"
"wazuh-states-vulnerabilities-*"
],
"order": 1,
"settings": {
Expand All @@ -15,15 +15,14 @@
"number_of_replicas": "0",
"refresh_interval": "2s",
"query.default_field": [
"base.tags",
"agent.id",
"host.os.family",
"host.os.full.text",
"host.os.full",
"host.os.version",
"package.name",
"package.version",
"vulnerability.id",
"vulnerability.description.text",
"vulnerability.description",
"vulnerability.severity",
"wazuh.cluster.name"
]
Expand Down
7 changes: 3 additions & 4 deletions ecs/vulnerability-detector/fields/template-settings.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"index_patterns": [
"wazuh-states-vulnerabilities"
"wazuh-states-vulnerabilities-*"
],
"priority": 1,
"template": {
Expand All @@ -16,15 +16,14 @@
"number_of_replicas": "0",
"refresh_interval": "2s",
"query.default_field": [
"base.tags",
"agent.id",
"host.os.family",
"host.os.full.text",
"host.os.full",
"host.os.version",
"package.name",
"package.version",
"vulnerability.id",
"vulnerability.description.text",
"vulnerability.description",
"vulnerability.severity",
"wazuh.cluster.name"
]
Expand Down

0 comments on commit e7f83c3

Please sign in to comment.