From 7790d432cc507d6ff7688c5b0ae411cf2c0f3d3d Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 13 May 2024 14:29:03 -0500 Subject: [PATCH] [FR] Bundle KQL & Kibana libs into base dependencies (#3662) (cherry picked from commit 78837549e8b20729153871449e9fcac56beaa1a3) --- .github/workflows/backport.yml | 1 - .github/workflows/get-target-branches.yml | 1 - .github/workflows/lock-versions.yml | 1 - .github/workflows/manual-backport.yml | 1 - .github/workflows/pythonpackage.yml | 1 - .github/workflows/release-docs.yml | 1 - .github/workflows/release-fleet.yml | 1 - CLI.md | 23 ++++++++++++----------- Makefile | 7 +------ README.md | 7 ++++--- detection_rules/etc/test_cli.bash | 2 +- pyproject.toml | 4 +++- 12 files changed, 21 insertions(+), 29 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b5203b14183..7eb66505051 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -99,7 +99,6 @@ jobs: run: | python -m pip install --upgrade pip pip install .[dev] - pip install lib/kql lib/kibana - name: Prune non-${{matrix.target_branch}} rules env: diff --git a/.github/workflows/get-target-branches.yml b/.github/workflows/get-target-branches.yml index f30597faff0..9542564a0b3 100644 --- a/.github/workflows/get-target-branches.yml +++ b/.github/workflows/get-target-branches.yml @@ -26,7 +26,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - id: get-branch-list run: | diff --git a/.github/workflows/lock-versions.yml b/.github/workflows/lock-versions.yml index 66f68023592..f46c97e46b6 100644 --- a/.github/workflows/lock-versions.yml +++ b/.github/workflows/lock-versions.yml @@ -36,7 +36,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Build release package run: | diff --git a/.github/workflows/manual-backport.yml b/.github/workflows/manual-backport.yml index cda9162b264..b2e0cb90603 100644 --- a/.github/workflows/manual-backport.yml +++ b/.github/workflows/manual-backport.yml @@ -55,7 +55,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Prune non-"${{github.event.inputs.target_branch}}" rules env: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8a787f9ab05..550d4355be2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Python Lint run: | diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml index 3c759182016..9f16212ad8b 100644 --- a/.github/workflows/release-docs.yml +++ b/.github/workflows/release-docs.yml @@ -48,7 +48,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Build Integration Docs env: diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index 36e0be9c4f8..bb5af132fb6 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -84,7 +84,6 @@ jobs: python -m pip install --upgrade pip pip cache purge pip install .[dev] - pip install lib/kql lib/kibana - name: Bump prebuilt rules package version env: diff --git a/CLI.md b/CLI.md index f7da7b7e009..18ce90ee4f8 100644 --- a/CLI.md +++ b/CLI.md @@ -43,7 +43,7 @@ Using the environment variable `DR_BYPASS_TAGS_VALIDATION` will bypass the Detec ## Importing rules into the repo -You can import rules into the repo using the `create-rule` or `import-rules` commands. Both of these commands will +You can import rules into the repo using the `create-rule` or `import-rules-to-repo` commands. Both of these commands will require that the rules are schema-compliant and able to pass full validation. The biggest benefit to using these commands is that they will strip[*](#note) additional fields[**](#note-2) and prompt for missing required fields. @@ -76,10 +76,10 @@ and will accept any valid rule in the following formats: * yaml (yup) * ndjson (as long as it contains only a single rule and has the extension `.ndjson` or `.jsonl`) -#### `import-rules` +#### `import-rules-to-repo` ```console -Usage: detection_rules import-rules [OPTIONS] [INPUT_FILE]... +Usage: detection_rules import-rules-to-repo [OPTIONS] [INPUT_FILE]... Import rules from json, toml, yaml, or Kibana exported rule file(s). @@ -269,22 +269,23 @@ Alternatively, rules can be exported into a consolidated ndjson file which can b directly. ```console -Usage: detection_rules export-rules [OPTIONS] +Usage: detection_rules export-rules-from-repo [OPTIONS] Export rule(s) into an importable ndjson file. Options: -f, --rule-file FILE - -d, --directory DIRECTORY Recursively export rules from a directory + -d, --directory DIRECTORY Recursively load rules from a directory -id, --rule-id TEXT - -o, --outfile FILE Name of file for exported rules + -o, --outfile PATH Name of file for exported rules -r, --replace-id Replace rule IDs with new IDs before export - --stack-version [7.8|7.9|7.10|7.11|7.12] + --stack-version [7.10|7.11|7.12|7.13|7.14|7.15|7.16|7.8|7.9|8.0|8.1|8.10|8.11|8.12|8.13|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9] Downgrade a rule version to be compatible with older instances of Kibana -s, --skip-unsupported If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise) + --include-metadata Add metadata to the exported rules -h, --help Show this message and exit. ``` @@ -336,7 +337,7 @@ Options: Example usage of a successful upload: ``` -python -m detection_rules kibana import-rules -f test-export-rules/credential_access_NEW_RULE.toml +python -m detection_rules kibana import-rules -f test-export-rules/credential_access_NEW_RULE.toml █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ █ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ @@ -375,7 +376,7 @@ python -m detection_rules kibana import-rules -f test-export-rules/credential_ac The rule loader detects a collision in name and fails as intended: ``` -python -m detection_rules kibana import-rules -d test-export-rules +python -m detection_rules kibana import-rules -d test-export-rules █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄ ▄ █▀▀▄ ▄ ▄ ▄ ▄▄▄ ▄▄▄ █ █ █▄▄ █ █▄▄ █ █ █ █ █ █▀▄ █ █▄▄▀ █ █ █ █▄▄ █▄▄ @@ -533,7 +534,7 @@ web_application_suspicious_activity_unauthorized_method.toml.toml Output of the `_errors.txt` file: ``` -cat test-export-rules/_errors.txt +cat test-export-rules/_errors.txt - Stolen Credentials Used to Login to Okta Account After MFA Reset - {'_schema': ['Setup header found in both note and setup fields.']} - First Occurrence of Okta User Session Started via Proxy - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]} - ESQL test: cmd child of Explorer - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'language': ['Must be equal to eql.']}), ValidationError({'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to threshold.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to new_terms.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'new_terms': ['Missing data for required field.']})]} @@ -552,7 +553,7 @@ Unknown field data_stream.dataset:osquery_manager.result and osquery_meta.counter>0 and osquery_meta.type:diff and osquery.last_run_code:0 and osquery_meta.action:removed ^^^^^^^^^^^^^^^^^ stack: 8.9.0, beats: 8.9.0, ecs: 8.9.0 -- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜ detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗ +- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜ detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗ ``` diff --git a/Makefile b/Makefile index ce5dc9b4456..19597c4cab5 100644 --- a/Makefile +++ b/Makefile @@ -20,15 +20,10 @@ clean: rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__ lib/kql/build lib/kibana/build lib/kql/*.egg-info lib/kibana/*.egg-info .PHONY: deps -deps: $(VENV) install-packages +deps: $(VENV) @echo "Installing all dependencies..." $(PIP) install .[dev] -.PHONY: install-packages -install-packages: - @echo "Installing kql and kibana packages..." - $(PIP) install lib/kql lib/kibana - .PHONY: pytest pytest: $(VENV) deps $(PYTHON) -m detection_rules test diff --git a/README.md b/README.md index ac0fdbac7b8..427a172c6ec 100644 --- a/README.md +++ b/README.md @@ -76,16 +76,17 @@ Collecting Click==7.0 Downloading Click-7.0-py2.py3-none-any.whl (81 kB) |████████████████████████████████| 81 kB 2.6 MB/s ... -pip3 install packages/kibana packages/kql ``` -Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `packages` directory or `git`. +Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `lib` directory. ```console + +# Install from the repository pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kibana pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kql -# or locally +# Or locally for development pip3 install lib/kibana lib/kql ``` diff --git a/detection_rules/etc/test_cli.bash b/detection_rules/etc/test_cli.bash index 87717cc7b23..1a865f8119a 100755 --- a/detection_rules/etc/test_cli.bash +++ b/detection_rules/etc/test_cli.bash @@ -15,7 +15,7 @@ echo "Viewing rule: threat_intel_indicator_match_address.toml" python -m detection_rules view-rule rules/cross-platform/threat_intel_indicator_match_address.toml echo "Exporting rule by ID: 0a97b20f-4144-49ea-be32-b540ecc445de" -python -m detection_rules export-rules --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de +python -m detection_rules export-rules-from-repo --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de echo "Updating rule data schemas" python -m detection_rules dev schemas update-rule-data diff --git a/pyproject.toml b/pyproject.toml index c07a6ceef67..623f10d9b3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,9 @@ dependencies = [ "typing-inspect==0.9.0", "typing-extensions==4.10.0", "XlsxWriter~=3.2.0", - "semver==3.0.2" + "semver==3.0.2", + "detection-rules-kql @ git+https://github.com/elastic/detection-rules.git#subdirectory=lib/kql", + "detection-rules-kibana @ git+https://github.com/elastic/detection-rules.git#subdirectory=lib/kibana" ] [project.optional-dependencies] dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=8.1.1", "pre-commit==3.6.2"]