[FR] Bundle KQL & Kibana libs into base dependencies (#3662)

(cherry picked from commit 7883754)

.github/workflows/backport.yml

@@ -99,7 +99,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .[dev]
-          pip install lib/kql lib/kibana
 
       - name: Prune non-${{matrix.target_branch}} rules
         env:

.github/workflows/get-target-branches.yml

@@ -26,7 +26,6 @@ jobs:
           python -m pip install --upgrade pip
           pip cache purge
           pip install .[dev]
-          pip install lib/kql lib/kibana
 
       - id: get-branch-list
         run: |

.github/workflows/lock-versions.yml

@@ -36,7 +36,6 @@ jobs:
           python -m pip install --upgrade pip
           pip cache purge
           pip install .[dev]
-          pip install lib/kql lib/kibana
 
       - name: Build release package
         run: |

.github/workflows/manual-backport.yml

@@ -55,7 +55,6 @@ jobs:
           python -m pip install --upgrade pip
           pip cache purge
           pip install .[dev]
-          pip install lib/kql lib/kibana
 
       - name: Prune non-"${{github.event.inputs.target_branch}}" rules
         env:

.github/workflows/pythonpackage.yml

@@ -24,7 +24,6 @@ jobs:
         python -m pip install --upgrade pip
         pip cache purge
         pip install .[dev]
-        pip install lib/kql lib/kibana
 
     - name: Python Lint
       run: |

.github/workflows/release-docs.yml

@@ -48,7 +48,6 @@ jobs:
           python -m pip install --upgrade pip
           pip cache purge
           pip install .[dev]
-          pip install lib/kql lib/kibana
 
       - name: Build Integration Docs
         env:

.github/workflows/release-fleet.yml

@@ -84,7 +84,6 @@ jobs:
             python -m pip install --upgrade pip
             pip cache purge
             pip install .[dev]
-            pip install lib/kql lib/kibana
 
         - name: Bump prebuilt rules package version
           env:

CLI.md

@@ -43,7 +43,7 @@ Using the environment variable `DR_BYPASS_TAGS_VALIDATION` will bypass the Detec
 
 ## Importing rules into the repo
 
-You can import rules into the repo using the `create-rule` or `import-rules` commands. Both of these commands will
+You can import rules into the repo using the `create-rule` or `import-rules-to-repo` commands. Both of these commands will
 require that the rules are schema-compliant and able to pass full validation. The biggest benefit to using these
 commands is that they will strip[*](#note) additional fields[**](#note-2) and prompt for missing required
 fields.
@@ -76,10 +76,10 @@ and will accept any valid rule in the following formats:
 * yaml (yup)
 * ndjson (as long as it contains only a single rule and has the extension `.ndjson` or `.jsonl`)
 
-#### `import-rules`
+#### `import-rules-to-repo`
 
 ```console
-Usage: detection_rules import-rules [OPTIONS] [INPUT_FILE]...
+Usage: detection_rules import-rules-to-repo [OPTIONS] [INPUT_FILE]...
 
   Import rules from json, toml, yaml, or Kibana exported rule file(s).
 
@@ -269,22 +269,23 @@ Alternatively, rules can be exported into a consolidated ndjson file which can b
 directly.
 
 ```console
-Usage: detection_rules export-rules [OPTIONS]
+Usage: detection_rules export-rules-from-repo [OPTIONS]
 
   Export rule(s) into an importable ndjson file.
 
 Options:
   -f, --rule-file FILE
-  -d, --directory DIRECTORY       Recursively export rules from a directory
+  -d, --directory DIRECTORY       Recursively load rules from a directory
   -id, --rule-id TEXT
-  -o, --outfile FILE              Name of file for exported rules
+  -o, --outfile PATH              Name of file for exported rules
   -r, --replace-id                Replace rule IDs with new IDs before export
-  --stack-version [7.8|7.9|7.10|7.11|7.12]
+  --stack-version [7.10|7.11|7.12|7.13|7.14|7.15|7.16|7.8|7.9|8.0|8.1|8.10|8.11|8.12|8.13|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9]
                                   Downgrade a rule version to be compatible
                                   with older instances of Kibana
   -s, --skip-unsupported          If `--stack-version` is passed, skip rule
                                   types which are unsupported (an error will
                                   be raised otherwise)
+  --include-metadata              Add metadata to the exported rules
   -h, --help                      Show this message and exit.
 ```
 
@@ -336,7 +337,7 @@ Options:
 Example usage of a successful upload:
 
 ```
-python -m detection_rules kibana import-rules -f test-export-rules/credential_access_NEW_RULE.toml        
+python -m detection_rules kibana import-rules -f test-export-rules/credential_access_NEW_RULE.toml
 
 █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄   ▄      █▀▀▄ ▄  ▄ ▄   ▄▄▄ ▄▄▄
 █  █ █▄▄  █  █▄▄ █    █   █  █ █ █▀▄ █      █▄▄▀ █  █ █   █▄▄ █▄▄
@@ -375,7 +376,7 @@ python -m detection_rules kibana import-rules -f test-export-rules/credential_ac
 
 The rule loader detects a collision in name and fails as intended:
 ```
-python -m detection_rules kibana import-rules -d test-export-rules                               
+python -m detection_rules kibana import-rules -d test-export-rules
 
 █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄   ▄      █▀▀▄ ▄  ▄ ▄   ▄▄▄ ▄▄▄
 █  █ █▄▄  █  █▄▄ █    █   █  █ █ █▀▄ █      █▄▄▀ █  █ █   █▄▄ █▄▄
@@ -533,7 +534,7 @@ web_application_suspicious_activity_unauthorized_method.toml.toml
 Output of the `_errors.txt` file:
 
 ```
-cat test-export-rules/_errors.txt 
+cat test-export-rules/_errors.txt
 - Stolen Credentials Used to Login to Okta Account After MFA Reset - {'_schema': ['Setup header found in both note and setup fields.']}
 - First Occurrence of Okta User Session Started via Proxy - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}
 - ESQL test: cmd child of Explorer - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'language': ['Must be equal to eql.']}), ValidationError({'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to threshold.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}}), ValidationError({'type': ['Must be equal to new_terms.'], 'threat': {0: {'tactic': {'reference': ['String does not match expected pattern.']}, 'technique': {0: {'reference': ['String does not match expected pattern.']}}}}, 'new_terms': ['Missing data for required field.']})]}
@@ -552,7 +553,7 @@ Unknown field
 data_stream.dataset:osquery_manager.result and osquery_meta.counter>0 and osquery_meta.type:diff and osquery.last_run_code:0 and osquery_meta.action:removed
                                                                           ^^^^^^^^^^^^^^^^^
 stack: 8.9.0, beats: 8.9.0, ecs: 8.9.0
-- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜  detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗ 
+- name - {'rule': [ValidationError({'type': ['Must be equal to eql.'], 'language': ['Must be equal to eql.']}), ValidationError({'type': ['Must be equal to esql.'], 'language': ['Must be equal to esql.']}), ValidationError({'type': ['Must be equal to threshold.'], 'threshold': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to threat_match.'], 'threat_mapping': ['Missing data for required field.'], 'threat_index': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to machine_learning.'], 'anomaly_threshold': ['Missing data for required field.'], 'machine_learning_job_id': ['Missing data for required field.']}), ValidationError({'type': ['Must be equal to query.']}), ValidationError({'new_terms': ['Missing data for required field.']})]}(venv312) ➜  detection-rules-fork git:(refresh-kibana-module-with-new-APIs) ✗
 ```
 
 

Makefile

@@ -20,15 +20,10 @@ clean:
 	rm -rf $(VENV) *.egg-info .eggs .egg htmlcov build dist packages .build .tmp .tox __pycache__  lib/kql/build lib/kibana/build lib/kql/*.egg-info lib/kibana/*.egg-info
 
 .PHONY: deps
-deps: $(VENV) install-packages
+deps: $(VENV)
 	@echo "Installing all dependencies..."
 	$(PIP) install .[dev]
 
-.PHONY: install-packages
-install-packages:
-	@echo "Installing kql and kibana packages..."
-	$(PIP) install lib/kql lib/kibana
-
 .PHONY: pytest
 pytest: $(VENV) deps
 	$(PYTHON) -m detection_rules test

README.md

@@ -76,16 +76,17 @@ Collecting Click==7.0
   Downloading Click-7.0-py2.py3-none-any.whl (81 kB)
      |████████████████████████████████| 81 kB 2.6 MB/s
 ...
-pip3 install packages/kibana packages/kql
 ```
 
-Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `packages` directory or `git`.
+Note: The `kibana` and `kql` packages are not available on PyPI and must be installed from the `lib` directory.
 
 ```console
+
+# Install from the repository
 pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kibana
 pip3 install git+https://github.com/elastic/detection-rules.git#subdirectory=kql
 
-# or locally
+# Or locally for development
 pip3 install lib/kibana lib/kql
 ```
 

detection_rules/etc/test_cli.bash

@@ -15,7 +15,7 @@ echo "Viewing rule: threat_intel_indicator_match_address.toml"
 python -m detection_rules view-rule rules/cross-platform/threat_intel_indicator_match_address.toml
 
 echo "Exporting rule by ID: 0a97b20f-4144-49ea-be32-b540ecc445de"
-python -m detection_rules export-rules --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de
+python -m detection_rules export-rules-from-repo --rule-id 0a97b20f-4144-49ea-be32-b540ecc445de
 
 echo "Updating rule data schemas"
 python -m detection_rules dev schemas update-rule-data

pyproject.toml

@@ -37,7 +37,9 @@ dependencies = [
 	"typing-inspect==0.9.0",
 	"typing-extensions==4.10.0",
 	"XlsxWriter~=3.2.0",
-	"semver==3.0.2"
+	"semver==3.0.2",
+	"detection-rules-kql @ git+https://github.com/elastic/detection-rules.git#subdirectory=lib/kql",
+	"detection-rules-kibana @ git+https://github.com/elastic/detection-rules.git#subdirectory=lib/kibana"
 ]
 [project.optional-dependencies]
 dev = ["pep8-naming==0.13.0", "PyGithub==2.2.0", "flake8==7.0.0", "pyflakes==3.2.0", "pytest>=8.1.1", "pre-commit==3.6.2"]