From f6ac2cf8603ca633070e719f69b4fcef45ea92cb Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Wed, 20 Nov 2024 13:36:37 +0100 Subject: [PATCH] [Rules migration] Add rules migrations update route (#11209) (#200815) ## Summary Changes in this PR: * Added `update` route to handle bulk rule migrations docs updates * Exposed `id` field in `RuleMigration` object needed for ES bulk update operation * Updated SIEM migrations schemas to use `NonEmptyString` when it is needed ## Testing locally Enable the flag ``` xpack.securitySolution.enableExperimental: ['siemMigrationsEnabled'] ``` Create and start a rule migration. Then use `update` API to updated corresponding docs. cURL request examples:
Rules migration `create` POST request ``` curl --location --request POST 'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules' \ --header 'kbn-xsrf;' \ --header 'x-elastic-internal-origin: security-solution' \ --header 'elastic-api-version: 1' \ --header 'Content-Type: application/json' \ --data '[ { "id": "f8c325ea-506e-4105-8ccf-da1492e90115", "vendor": "splunk", "title": "Linux Auditd Add User Account Type", "description": "The following analytic detects the suspicious add user account type. This behavior is critical for a SOC to monitor because it may indicate attempts to gain unauthorized access or maintain control over a system. Such actions could be signs of malicious activity. If confirmed, this could lead to serious consequences, including a compromised system, unauthorized access to sensitive data, or even a wider breach affecting the entire network. Detecting and responding to these signs early is essential to prevent potential security incidents.", "query": "sourcetype=\"linux:audit\" type=ADD_USER \n| rename hostname as dest \n| stats count min(_time) as firstTime max(_time) as lastTime by exe pid dest res UID type \n| `security_content_ctime(firstTime)` \n| `security_content_ctime(lastTime)`\n| search *", "query_language":"spl", "mitre_attack_ids": [ "T1136" ] }, { "id": "7b87c556-0ca4-47e0-b84c-6cd62a0a3e90", "vendor": "splunk", "title": "Linux Auditd Change File Owner To Root", "description": "The following analytic detects the use of the '\''chown'\'' command to change a file owner to '\''root'\'' on a Linux system. It leverages Linux Auditd telemetry, specifically monitoring command-line executions and process details. This activity is significant as it may indicate an attempt to escalate privileges by adversaries, malware, or red teamers. If confirmed malicious, this action could allow an attacker to gain root-level access, leading to full control over the compromised host and potential persistence within the environment.", "query": "`linux_auditd` `linux_auditd_normalized_proctitle_process`\r\n| rename host as dest \r\n| where LIKE (process_exec, \"%chown %root%\") \r\n| stats count min(_time) as firstTime max(_time) as lastTime by process_exec proctitle normalized_proctitle_delimiter dest \r\n| `security_content_ctime(firstTime)` \r\n| `security_content_ctime(lastTime)`\r\n| `linux_auditd_change_file_owner_to_root_filter`", "query_language": "spl", "mitre_attack_ids": [ "T1222" ] } ]' ```
Rules migration `start` task request - Assuming the connector `azureOpenAiGPT4o` is already created in the local environment. - Using the {{`migration_id`}} from the first POST request response ``` curl --location --request PUT 'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules/{{migration_id}}/start' \ --header 'kbn-xsrf;' \ --header 'x-elastic-internal-origin: security-solution' \ --header 'elastic-api-version: 1' \ --header 'Content-Type: application/json' \ --data '{ "connectorId": "azureOpenAiGPT4o" }' ```
Rules migration rules documents request - Using the {{`migration_id`}} from the first POST request response. ``` curl --location --request GET 'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules/{{migration_id}}' \ --header 'kbn-xsrf;' \ --header 'x-elastic-internal-origin: security-solution' \ --header 'elastic-api-version: 1' ```
Rules migration `update` PUT request - Using the {{`rule_migration_id_1`}} and {{`rule_migration_id_2`}} from previous GET request response ``` curl --location --request PUT 'http://elastic:changeme@localhost:5601/internal/siem_migrations/rules' \ --header 'kbn-xsrf;' \ --header 'x-elastic-internal-origin: security-solution' \ --header 'elastic-api-version: 1' --data '[ { "comments": [ "## Migration Summary\n- The `FROM` command is used to select the `logs-*` index pattern.\n- The `RENAME` command is used to rename the `host` field to `dest`.\n- The `WHERE` command filters the rows where `process_exec` contains the pattern `*chown *root*`.\n- The `STATS` command is used to aggregate the data, counting the number of occurrences and finding the minimum and maximum timestamps, grouped by `process_exec`, `proctitle`, `normalized_proctitle_delimiter`, and `dest`.\n- The macros `security_content_ctime` and `linux_auditd_change_file_owner_to_root_filter` are placeholders for the corresponding Splunk macros.", "Additional comment 2.0" ], "translation_result": "full", "id": "{{rule_migration_id_1}}" }, { "created_by": "elastic2.0", "elastic_rule": { "severity": "high", "title": "Linux Auditd Change File Owner To Root (UPDATED)" }, "id": "{{rule_migration_id_2}}" } ]' ```
--------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/api/quickstart_client.gen.ts | 21 +++++ .../model/api/rules/rule_migration.gen.ts | 61 +++++++++++-- .../api/rules/rule_migration.schema.yaml | 73 +++++++++++++--- .../model/{api => }/common.gen.ts | 11 ++- .../model/{api => }/common.schema.yaml | 7 +- .../model/rule_migration.gen.ts | 82 +++++++++++++++--- .../model/rule_migration.schema.yaml | 85 +++++++++++++------ .../lib/siem_migrations/rules/api/create.ts | 2 +- .../lib/siem_migrations/rules/api/index.ts | 2 + .../lib/siem_migrations/rules/api/update.ts | 52 ++++++++++++ .../data/rule_migrations_data_base_client.ts | 2 +- .../rules/data/rule_migrations_data_client.ts | 8 -- .../data/rule_migrations_data_rules_client.ts | 58 +++++++++++-- .../rules/data/rule_migrations_field_maps.ts | 6 +- .../rules/task/rule_migrations_task_client.ts | 6 +- .../server/lib/siem_migrations/rules/types.ts | 2 +- .../services/security_solution_api.gen.ts | 15 ++++ 17 files changed, 408 insertions(+), 85 deletions(-) rename x-pack/plugins/security_solution/common/siem_migrations/model/{api => }/common.gen.ts (77%) rename x-pack/plugins/security_solution/common/siem_migrations/model/{api => }/common.schema.yaml (71%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 513e2163f932f..f0d8445c41eed 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -371,6 +371,8 @@ import type { StartRuleMigrationResponse, StopRuleMigrationRequestParamsInput, StopRuleMigrationResponse, + UpdateRuleMigrationRequestBodyInput, + UpdateRuleMigrationResponse, UpsertRuleMigrationResourcesRequestParamsInput, UpsertRuleMigrationResourcesRequestBodyInput, UpsertRuleMigrationResourcesResponse, @@ -2099,6 +2101,22 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Updates rules migrations attributes + */ + async updateRuleMigration(props: UpdateRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API UpdateRuleMigration`); + return this.kbnClient + .request({ + path: '/internal/siem_migrations/rules', + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + body: props.body, + }) + .catch(catchAxiosErrorFormatAndThrow); + } async uploadAssetCriticalityRecords(props: UploadAssetCriticalityRecordsProps) { this.log.info(`${new Date().toISOString()} Calling API UploadAssetCriticalityRecords`); return this.kbnClient @@ -2401,6 +2419,9 @@ export interface TriggerRiskScoreCalculationProps { export interface UpdateRuleProps { body: UpdateRuleRequestBodyInput; } +export interface UpdateRuleMigrationProps { + body: UpdateRuleMigrationRequestBodyInput; +} export interface UploadAssetCriticalityRecordsProps { attachment: FormData; } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 7ea6314726dab..ac15080f2e0a4 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -19,6 +19,9 @@ import { ArrayFromString } from '@kbn/zod-helpers'; import { OriginalRule, + ElasticRulePartial, + RuleMigrationTranslationResult, + RuleMigrationComments, RuleMigrationAllTaskStats, RuleMigration, RuleMigrationTaskStats, @@ -26,7 +29,7 @@ import { RuleMigrationResourceType, RuleMigrationResource, } from '../../rule_migration.gen'; -import { ConnectorId, LangSmithOptions } from '../common.gen'; +import { NonEmptyString, ConnectorId, LangSmithOptions } from '../../common.gen'; export type CreateRuleMigrationRequestBody = z.infer; export const CreateRuleMigrationRequestBody = z.array(OriginalRule); @@ -37,7 +40,7 @@ export const CreateRuleMigrationResponse = z.object({ /** * The migration id created. */ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type GetAllStatsRuleMigrationResponse = z.infer; @@ -45,7 +48,7 @@ export const GetAllStatsRuleMigrationResponse = RuleMigrationAllTaskStats; export type GetRuleMigrationRequestParams = z.infer; export const GetRuleMigrationRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type GetRuleMigrationRequestParamsInput = z.input; @@ -66,7 +69,7 @@ export type GetRuleMigrationResourcesRequestParams = z.infer< typeof GetRuleMigrationResourcesRequestParams >; export const GetRuleMigrationResourcesRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type GetRuleMigrationResourcesRequestParamsInput = z.input< typeof GetRuleMigrationResourcesRequestParams @@ -77,7 +80,7 @@ export const GetRuleMigrationResourcesResponse = z.array(RuleMigrationResource); export type GetRuleMigrationStatsRequestParams = z.infer; export const GetRuleMigrationStatsRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type GetRuleMigrationStatsRequestParamsInput = z.input< typeof GetRuleMigrationStatsRequestParams @@ -88,7 +91,7 @@ export const GetRuleMigrationStatsResponse = RuleMigrationTaskStats; export type StartRuleMigrationRequestParams = z.infer; export const StartRuleMigrationRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type StartRuleMigrationRequestParamsInput = z.input; @@ -109,7 +112,7 @@ export const StartRuleMigrationResponse = z.object({ export type StopRuleMigrationRequestParams = z.infer; export const StopRuleMigrationRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type StopRuleMigrationRequestParamsInput = z.input; @@ -121,11 +124,42 @@ export const StopRuleMigrationResponse = z.object({ stopped: z.boolean(), }); +export type UpdateRuleMigrationRequestBody = z.infer; +export const UpdateRuleMigrationRequestBody = z.array( + z.object({ + /** + * The rule migration id + */ + id: NonEmptyString, + /** + * The migrated elastic rule attributes to update. + */ + elastic_rule: ElasticRulePartial.optional(), + /** + * The rule translation result. + */ + translation_result: RuleMigrationTranslationResult.optional(), + /** + * The comments for the migration including a summary from the LLM in markdown. + */ + comments: RuleMigrationComments.optional(), + }) +); +export type UpdateRuleMigrationRequestBodyInput = z.input; + +export type UpdateRuleMigrationResponse = z.infer; +export const UpdateRuleMigrationResponse = z.object({ + /** + * Indicates rules migrations have been updated. + */ + updated: z.boolean(), +}); + export type UpsertRuleMigrationResourcesRequestParams = z.infer< typeof UpsertRuleMigrationResourcesRequestParams >; export const UpsertRuleMigrationResourcesRequestParams = z.object({ - migration_id: z.string(), + migration_id: NonEmptyString, }); export type UpsertRuleMigrationResourcesRequestParamsInput = z.input< typeof UpsertRuleMigrationResourcesRequestParams @@ -134,7 +168,16 @@ export type UpsertRuleMigrationResourcesRequestParamsInput = z.input< export type UpsertRuleMigrationResourcesRequestBody = z.infer< typeof UpsertRuleMigrationResourcesRequestBody >; -export const UpsertRuleMigrationResourcesRequestBody = z.array(RuleMigrationResourceData); +export const UpsertRuleMigrationResourcesRequestBody = z.array( + RuleMigrationResourceData.merge( + z.object({ + /** + * The rule resource migration id + */ + id: NonEmptyString, + }) + ) +); export type UpsertRuleMigrationResourcesRequestBodyInput = z.input< typeof UpsertRuleMigrationResourcesRequestBody >; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index bac82e5b0248e..7785304671129 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -3,7 +3,6 @@ info: title: SIEM Rules Migration API version: '1' paths: - # Rule migrations APIs /internal/siem_migrations/rules: @@ -33,8 +32,52 @@ paths: - migration_id properties: migration_id: - type: string description: The migration id created. + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' + + put: + summary: Updates rules migrations + operationId: UpdateRuleMigration + x-codegen-enabled: true + description: Updates rules migrations attributes + tags: + - SIEM Rule Migrations + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: object + required: + - id + properties: + id: + description: The rule migration id + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' + elastic_rule: + description: The migrated elastic rule attributes to update. + $ref: '../../rule_migration.schema.yaml#/components/schemas/ElasticRulePartial' + translation_result: + description: The rule translation result. + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationTranslationResult' + comments: + description: The comments for the migration including a summary from the LLM in markdown. + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationComments' + responses: + 200: + description: Indicates rules migrations have been updated correctly. + content: + application/json: + schema: + type: object + required: + - updated + properties: + updated: + type: boolean + description: Indicates rules migrations have been updated. /internal/siem_migrations/rules/stats: get: @@ -67,8 +110,8 @@ paths: in: path required: true schema: - type: string description: The migration id to start + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' responses: 200: description: Indicates rule migration have been retrieved correctly. @@ -94,8 +137,8 @@ paths: in: path required: true schema: - type: string description: The migration id to start + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' requestBody: required: true content: @@ -106,9 +149,9 @@ paths: - connector_id properties: connector_id: - $ref: '../common.schema.yaml#/components/schemas/ConnectorId' + $ref: '../../common.schema.yaml#/components/schemas/ConnectorId' langsmith_options: - $ref: '../common.schema.yaml#/components/schemas/LangSmithOptions' + $ref: '../../common.schema.yaml#/components/schemas/LangSmithOptions' responses: 200: description: Indicates the migration start request has been processed successfully. @@ -138,8 +181,8 @@ paths: in: path required: true schema: - type: string description: The migration id to start + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' responses: 200: description: Indicates the migration stats has been retrieved correctly. @@ -163,8 +206,8 @@ paths: in: path required: true schema: - type: string description: The migration id to stop + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' responses: 200: description: Indicates migration task stop has been processed successfully. @@ -197,8 +240,8 @@ paths: in: path required: true schema: - type: string description: The migration id to attach the resources + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' requestBody: required: true content: @@ -206,7 +249,15 @@ paths: schema: type: array items: - $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' + allOf: + - $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' + - type: object + required: + - id + properties: + id: + description: The rule resource migration id + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' responses: 200: description: Indicates migration resources have been created or updated correctly. @@ -234,8 +285,8 @@ paths: in: path required: true schema: - type: string description: The migration id to attach the resources + $ref: '../../common.schema.yaml#/components/schemas/NonEmptyString' - name: type in: query required: false diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/common.gen.ts similarity index 77% rename from x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts rename to x-pack/plugins/security_solution/common/siem_migrations/model/common.gen.ts index 7880354928538..9b1d0756c3a3b 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/common.gen.ts @@ -10,12 +10,21 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: SIEM Rule Migrations API common components + * title: SIEM Rule Migration common components * version: not applicable */ import { z } from '@kbn/zod'; +/** + * A string that is not empty and does not contain only whitespace + */ +export type NonEmptyString = z.infer; +export const NonEmptyString = z + .string() + .min(1) + .regex(/^(?! *$).+$/); + /** * The GenAI connector id to use. */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/common.schema.yaml similarity index 71% rename from x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml rename to x-pack/plugins/security_solution/common/siem_migrations/model/common.schema.yaml index 5782fa7772013..a50225df778ad 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/common.schema.yaml @@ -1,11 +1,16 @@ openapi: 3.0.3 info: - title: SIEM Rule Migrations API common components + title: SIEM Rule Migration common components version: 'not applicable' paths: {} components: x-codegen-enabled: true schemas: + NonEmptyString: + type: string + pattern: ^(?! *$).+$ + minLength: 1 + description: A string that is not empty and does not contain only whitespace ConnectorId: type: string description: The GenAI connector id to use. diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index ac178610cee62..0554ef18a13f7 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -10,12 +10,14 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: SIEM Rule Migration common components + * title: SIEM Rule Migration components * version: not applicable */ import { z } from '@kbn/zod'; +import { NonEmptyString } from './common.gen'; + /** * The original rule vendor identifier. */ @@ -30,7 +32,10 @@ export const OriginalRule = z.object({ /** * The original rule id. */ - id: z.string(), + id: NonEmptyString, + /** + * The original rule vendor identifier. + */ vendor: OriginalRuleVendor, /** * The original rule name. @@ -82,18 +87,46 @@ export const ElasticRule = z.object({ /** * The Elastic prebuilt rule id matched. */ - prebuilt_rule_id: z.string().optional(), + prebuilt_rule_id: NonEmptyString.optional(), /** * The Elastic rule id installed as a result. */ - id: z.string().optional(), + id: NonEmptyString.optional(), }); +/** + * The partial version of the migrated elastic rule. + */ +export type ElasticRulePartial = z.infer; +export const ElasticRulePartial = ElasticRule.partial(); + +/** + * The rule translation result. + */ +export type RuleMigrationTranslationResult = z.infer; +export const RuleMigrationTranslationResult = z.enum(['full', 'partial', 'untranslatable']); +export type RuleMigrationTranslationResultEnum = typeof RuleMigrationTranslationResult.enum; +export const RuleMigrationTranslationResultEnum = RuleMigrationTranslationResult.enum; + +/** + * The status of the rule migration process. + */ +export type RuleMigrationStatus = z.infer; +export const RuleMigrationStatus = z.enum(['pending', 'processing', 'completed', 'failed']); +export type RuleMigrationStatusEnum = typeof RuleMigrationStatus.enum; +export const RuleMigrationStatusEnum = RuleMigrationStatus.enum; + +/** + * The comments for the migration including a summary from the LLM in markdown. + */ +export type RuleMigrationComments = z.infer; +export const RuleMigrationComments = z.array(z.string()); + /** * The rule migration document object. */ -export type RuleMigration = z.infer; -export const RuleMigration = z.object({ +export type RuleMigrationData = z.infer; +export const RuleMigrationData = z.object({ /** * The moment of creation */ @@ -101,25 +134,31 @@ export const RuleMigration = z.object({ /** * The migration id. */ - migration_id: z.string(), + migration_id: NonEmptyString, /** * The username of the user who created the migration. */ - created_by: z.string(), + created_by: NonEmptyString, + /** + * The original rule to migrate. + */ original_rule: OriginalRule, + /** + * The migrated elastic rule. + */ elastic_rule: ElasticRule.optional(), /** * The rule translation result. */ - translation_result: z.enum(['full', 'partial', 'untranslatable']).optional(), + translation_result: RuleMigrationTranslationResult.optional(), /** * The status of the rule migration process. */ - status: z.enum(['pending', 'processing', 'completed', 'failed']).default('pending'), + status: RuleMigrationStatus.default('pending'), /** * The comments for the migration including a summary from the LLM in markdown. */ - comments: z.array(z.string()).optional(), + comments: RuleMigrationComments.optional(), /** * The moment of the last update */ @@ -130,6 +169,19 @@ export const RuleMigration = z.object({ updated_by: z.string().optional(), }); +/** + * The rule migration document object. + */ +export type RuleMigration = z.infer; +export const RuleMigration = z + .object({ + /** + * The rule migration id + */ + id: NonEmptyString, + }) + .merge(RuleMigrationData); + /** * The rule migration task stats object. */ @@ -177,7 +229,7 @@ export const RuleMigrationAllTaskStats = z.array( /** * The migration id */ - migration_id: z.string(), + migration_id: NonEmptyString, }) ) ); @@ -216,10 +268,14 @@ export const RuleMigrationResourceData = z.object({ export type RuleMigrationResource = z.infer; export const RuleMigrationResource = RuleMigrationResourceData.merge( z.object({ + /** + * The rule resource migration id + */ + id: NonEmptyString, /** * The migration id */ - migration_id: z.string(), + migration_id: NonEmptyString, /** * The moment of the last update */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index c16849cec278f..95ff05df39a15 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -1,12 +1,11 @@ openapi: 3.0.3 info: - title: SIEM Rule Migration common components + title: SIEM Rule Migration components version: 'not applicable' paths: {} components: x-codegen-enabled: true schemas: - OriginalRuleVendor: type: string description: The original rule vendor identifier. @@ -25,9 +24,10 @@ components: - query_language properties: id: - type: string description: The original rule id. + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' vendor: + description: The original rule vendor identifier. $ref: '#/components/schemas/OriginalRuleVendor' title: type: string @@ -71,13 +71,30 @@ components: enum: - esql prebuilt_rule_id: - type: string description: The Elastic prebuilt rule id matched. + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' id: - type: string description: The Elastic rule id installed as a result. + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' + + ElasticRulePartial: + description: The partial version of the migrated elastic rule. + $ref: '#/components/schemas/ElasticRule' + x-modify: partial RuleMigration: + description: The rule migration document object. + allOf: + - type: object + required: + - id + properties: + id: + description: The rule migration id + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' + - $ref: '#/components/schemas/RuleMigrationData' + + RuleMigrationData: type: object description: The rule migration document object. required: @@ -91,36 +108,27 @@ components: type: string description: The moment of creation migration_id: - type: string description: The migration id. + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' created_by: - type: string description: The username of the user who created the migration. + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' original_rule: + description: The original rule to migrate. $ref: '#/components/schemas/OriginalRule' elastic_rule: + description: The migrated elastic rule. $ref: '#/components/schemas/ElasticRule' translation_result: - type: string description: The rule translation result. - enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts - - full - - partial - - untranslatable + $ref: '#/components/schemas/RuleMigrationTranslationResult' status: - type: string description: The status of the rule migration process. - enum: # should match SiemMigrationsStatus enum at ../constants.ts - - pending - - processing - - completed - - failed + $ref: '#/components/schemas/RuleMigrationStatus' default: pending comments: - type: array description: The comments for the migration including a summary from the LLM in markdown. - items: - type: string + $ref: '#/components/schemas/RuleMigrationComments' updated_at: type: string description: The moment of the last update @@ -182,11 +190,34 @@ components: - migration_id properties: migration_id: - type: string description: The migration id + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' + + RuleMigrationTranslationResult: + type: string + description: The rule translation result. + enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts + - full + - partial + - untranslatable + + RuleMigrationStatus: + type: string + description: The status of the rule migration process. + enum: # should match SiemMigrationsStatus enum at ../constants.ts + - pending + - processing + - completed + - failed + + RuleMigrationComments: + type: array + description: The comments for the migration including a summary from the LLM in markdown. + items: + type: string + + ## Rule migration resources -## Rule migration resources - RuleMigrationResourceType: type: string description: The type of the rule migration resource. @@ -220,11 +251,15 @@ components: - $ref: '#/components/schemas/RuleMigrationResourceData' - type: object required: + - id - migration_id properties: + id: + description: The rule resource migration id + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' migration_id: - type: string description: The migration id + $ref: './common.schema.yaml#/components/schemas/NonEmptyString' updated_at: type: string description: The moment of the last update diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index 025c52da766ad..a937560842f74 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -14,7 +14,7 @@ import { } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import type { CreateRuleMigrationInput } from '../data/rule_migrations_data_client'; +import type { CreateRuleMigrationInput } from '../data/rule_migrations_data_rules_client'; import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsCreateRoute = ( diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index dfc4c2156fe2d..c6ea6b8bf897b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -8,6 +8,7 @@ import type { Logger } from '@kbn/core/server'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; +import { registerSiemRuleMigrationsUpdateRoute } from './update'; import { registerSiemRuleMigrationsGetRoute } from './get'; import { registerSiemRuleMigrationsStartRoute } from './start'; import { registerSiemRuleMigrationsStatsRoute } from './stats'; @@ -22,6 +23,7 @@ export const registerSiemRuleMigrationsRoutes = ( logger: Logger ) => { registerSiemRuleMigrationsCreateRoute(router, logger); + registerSiemRuleMigrationsUpdateRoute(router, logger); registerSiemRuleMigrationsStatsAllRoute(router, logger); registerSiemRuleMigrationsGetRoute(router, logger); registerSiemRuleMigrationsStartRoute(router, logger); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts new file mode 100644 index 0000000000000..a41ba32d2dd34 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/update.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { + UpdateRuleMigrationRequestBody, + type UpdateRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; + +export const registerSiemRuleMigrationsUpdateRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { body: buildRouteValidationWithZod(UpdateRuleMigrationRequestBody) }, + }, + }, + withLicense( + async (context, req, res): Promise> => { + const rulesToUpdate = req.body; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + await ruleMigrationsClient.data.rules.update(rulesToUpdate); + + return res.ok({ body: { updated: true } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts index 8b5a81e2bc99d..4f0b65e063b77 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts @@ -33,7 +33,7 @@ export class RuleMigrationsDataBaseClient { return hits.map(({ _id, _source }) => { assert(_id, 'document should have _id'); assert(_source, 'document should have _source'); - return { ..._source, ...override, _id }; + return { ..._source, ...override, id: _id }; }); } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts index fe682ceeec783..40f4aa6bf786e 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts @@ -6,18 +6,10 @@ */ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { - RuleMigration, - RuleMigrationTaskStats, -} from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client'; import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client'; import type { AdapterId } from './rule_migrations_data_service'; -export type CreateRuleMigrationInput = Omit; -export type RuleMigrationDataStats = Omit; -export type RuleMigrationAllDataStats = Array; - export type IndexNameProvider = () => Promise; export type IndexNameProviders = Record; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index feedff65343d5..a01d36e9a1195 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -15,12 +15,20 @@ import type { import type { StoredRuleMigration } from '../types'; import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; import type { + ElasticRule, RuleMigration, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; -export type CreateRuleMigrationInput = Omit; +export type CreateRuleMigrationInput = Omit< + RuleMigration, + '@timestamp' | 'id' | 'status' | 'created_by' +>; +export type UpdateRuleMigrationInput = { elastic_rule?: Partial } & Pick< + RuleMigration, + 'id' | 'translation_result' | 'comments' +>; export type RuleMigrationDataStats = Omit; export type RuleMigrationAllDataStats = Array; @@ -35,6 +43,7 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient const index = await this.getIndexName(); let ruleMigrationsSlice: CreateRuleMigrationInput[]; + const createdAt = new Date().toISOString(); while ((ruleMigrationsSlice = ruleMigrations.splice(0, BULK_MAX_SIZE)).length) { await this.esClient .bulk({ @@ -43,9 +52,11 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient { create: { _index: index } }, { ...ruleMigration, - '@timestamp': new Date().toISOString(), + '@timestamp': createdAt, status: SiemMigrationStatus.PENDING, created_by: this.username, + updated_by: this.username, + updated_at: createdAt, }, ]), }) @@ -56,6 +67,37 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient } } + /** Updates an array of rule migrations to be processed */ + async update(ruleMigrations: UpdateRuleMigrationInput[]): Promise { + const index = await this.getIndexName(); + + let ruleMigrationsSlice: UpdateRuleMigrationInput[]; + const updatedAt = new Date().toISOString(); + while ((ruleMigrationsSlice = ruleMigrations.splice(0, BULK_MAX_SIZE)).length) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: ruleMigrationsSlice.flatMap((ruleMigration) => { + const { id, ...rest } = ruleMigration; + return [ + { update: { _index: index, _id: id } }, + { + doc: { + ...rest, + updated_by: this.username, + updated_at: updatedAt, + }, + }, + ]; + }), + }) + .catch((error) => { + this.logger.error(`Error updating rule migrations: ${error.message}`); + throw error; + }); + } + } + /** Retrieves an array of rule documents of a specific migrations */ async get(migrationId: string): Promise { const index = await this.getIndexName(); @@ -94,8 +136,8 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient await this.esClient .bulk({ refresh: 'wait_for', - operations: storedRuleMigrations.flatMap(({ _id, status }) => [ - { update: { _id, _index: index } }, + operations: storedRuleMigrations.flatMap(({ id, status }) => [ + { update: { _id: id, _index: index } }, { doc: { status, updated_by: this.username, updated_at: new Date().toISOString() }, }, @@ -112,7 +154,7 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient } /** Updates one rule migration with the provided data and sets the status to `completed` */ - async saveCompleted({ _id, ...ruleMigration }: StoredRuleMigration): Promise { + async saveCompleted({ id, ...ruleMigration }: StoredRuleMigration): Promise { const index = await this.getIndexName(); const doc = { ...ruleMigration, @@ -120,14 +162,14 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient updated_by: this.username, updated_at: new Date().toISOString(), }; - await this.esClient.update({ index, id: _id, doc, refresh: 'wait_for' }).catch((error) => { + await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { this.logger.error(`Error updating rule migration status to completed: ${error.message}`); throw error; }); } /** Updates one rule migration with the provided data and sets the status to `failed` */ - async saveError({ _id, ...ruleMigration }: StoredRuleMigration): Promise { + async saveError({ id, ...ruleMigration }: StoredRuleMigration): Promise { const index = await this.getIndexName(); const doc = { ...ruleMigration, @@ -135,7 +177,7 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient updated_by: this.username, updated_at: new Date().toISOString(), }; - await this.esClient.update({ index, id: _id, doc, refresh: 'wait_for' }).catch((error) => { + await this.esClient.update({ index, id, doc, refresh: 'wait_for' }).catch((error) => { this.logger.error(`Error updating rule migration status to failed: ${error.message}`); throw error; }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index 8dbccb61d5355..3811ff74b5ca1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -11,7 +11,7 @@ import type { RuleMigrationResource, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -export const ruleMigrationsFieldMap: FieldMap> = { +export const ruleMigrationsFieldMap: FieldMap>> = { '@timestamp': { type: 'date', required: false }, migration_id: { type: 'keyword', required: true }, created_by: { type: 'keyword', required: true }, @@ -38,7 +38,9 @@ export const ruleMigrationsFieldMap: FieldMap> updated_by: { type: 'keyword', required: false }, }; -export const ruleMigrationResourcesFieldMap: FieldMap> = { +export const ruleMigrationResourcesFieldMap: FieldMap< + SchemaFieldMapKeys> +> = { migration_id: { type: 'keyword', required: true }, type: { type: 'keyword', required: true }, name: { type: 'keyword', required: true }, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 98319a77a7662..989c33a44cb36 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -13,10 +13,8 @@ import type { RuleMigrationAllTaskStats, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { - RuleMigrationDataStats, - RuleMigrationsDataClient, -} from '../data/rule_migrations_data_client'; +import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client'; +import type { RuleMigrationDataStats } from '../data/rule_migrations_data_rules_client'; import type { RuleMigrationTaskStartParams, RuleMigrationTaskStartResult, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 34d0088256282..e506b43cc323b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -10,7 +10,7 @@ import type { RuleMigrationResource, } from '../../../../common/siem_migrations/model/rule_migration.gen'; -export type Stored = T & { _id: string }; +export type Stored = T & { id: string }; export type StoredRuleMigration = Stored; export type StoredRuleMigrationResource = Stored; diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index 6ba76b071d860..3574199709aee 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -140,6 +140,7 @@ import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plug import { SuggestUserProfilesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/users/suggest_user_profiles_route.gen'; import { TriggerRiskScoreCalculationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; import { UpdateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/update_rule/update_rule_route.gen'; +import { UpdateRuleMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { UpsertRuleMigrationResourcesRequestParamsInput, UpsertRuleMigrationResourcesRequestBodyInput, @@ -1434,6 +1435,17 @@ detection engine rules. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Updates rules migrations attributes + */ + updateRuleMigration(props: UpdateRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace)) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(props.body as object); + }, uploadAssetCriticalityRecords(kibanaSpace: string = 'default') { return supertest .post(routeWithNamespace('/api/asset_criticality/upload_csv', kibanaSpace)) @@ -1727,6 +1739,9 @@ export interface TriggerRiskScoreCalculationProps { export interface UpdateRuleProps { body: UpdateRuleRequestBodyInput; } +export interface UpdateRuleMigrationProps { + body: UpdateRuleMigrationRequestBodyInput; +} export interface UpsertRuleMigrationResourcesProps { params: UpsertRuleMigrationResourcesRequestParamsInput; body: UpsertRuleMigrationResourcesRequestBodyInput;