[SecuritySolution][EntityAnalytics] Risk Scoring Preview API (#155966)

## Summary

This PR adds a new Risk Scoring API endpoint. Its functionality is meant
to replace the current transform-based solution.

### Contents of this PR:
- New feature flag: `riskScoringRoutesEnabled`
- A new POST endpoint at `/internal/risk_scores/preview`
- An OpenAPI doc for the endpoint
- Unit and integration tests

### Current behavior, and short-term plans
The endpoint as specified in this branch is _read-only_. When the
endpoint is hit, it triggers some aggregations in elasticsearch, and a
formatted response is returned; there is no persistence at this time.
This endpoint was originally written as a POC to demonstrate the new
Risk Engine's functionality, but it will now drive the [Preview Risk
Scoring](elastic/security-team#6443) feature.

The main path for the Risk Engine is going to be a _scheduled task_ that
calculates Risk Scores and writes them to a persistent datastream that
we own. (elastic/security-team#6450). To
accomplish this, we will decompose the full functionality of this
endpoint into constituent pieces (i.e. `calculate | persist, get`)

## How to review
I've created a Postman collection that can be used to exercise this
endpoint. It was generated by Postman from the OpenAPI spec, and
modified by me to contain a valid subset of request parameters; please
peruse the spec and/or feel free to generate your own scripts/tools from
the spec.
```
curl -L -H 'Authorization: 10c7f646373aa116' -o 'Risk Scoring API.postman_collection.json' https://upload.elastic.co/d/007a57857fc40c791835629ea6dd692d2a8a290860f2917329d688be78c03b1d
```

### Review against the PR instance
I've created a [demo
instance](https://rylnd-pr-155966-risk-score-api.kbndev.co/) containing
the code on this branch, along with some realistic(ish) alert data
(~200k alerts). While you can use this instance as a convenience, you
will need to [set up
kibana-remote-dev](https://github.com/elastic/kibana-remote-dev#access-kibana-es-locally-without-sso)
and forward ports in order to be able to access the instance's API from
a local machine:

1. Configure kibana-remote-dev with your SSH key and GitHub token.
2. Configure kibana-remote-dev to specify `GITHUB_USERNAME=rylnd`
* This allows you to bypass kibana-remote-dev code that assumes projects
are owned by you
3. Forward local ports to my instance: `./ports
rd-rylnd-pr-155966-risk-score-api`
4. Use postman to talk to `http://localhost:5601`, which will be
forwarded to the cloud instance via the previous command

### Review manually
1. Check out this branch
3. Enable the feature flag
4. Populate some event data and generate some alerts
5. Navigate to the new endpoint, and observe that the `host.name`s and
`user.name`s from those alerts have been aggregated into these "risk
scores" in the response
6. Play with the request options to see how these affect the scores (and
see docs/test for more details on how those work)

## _What_ to review
* Are the scores internally consistent? I.e. do they add up as expected?
Does the corresponding "level" make sense?
* Do parameters apply as expected? E.g. do weights predictably scale the
results?
* Are there discrepancies between the spec and the actual
implementation?
* Does pagination make sense? (i.e. the `after_keys` stuff)?

#### TODO (for @rylnd)
- [x] Add `description`s to the OpenAPI docs
- [x] Remove remaining TODOs from code


### Checklist


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)




### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

Related ticket: elastic/security-team#4211

---------

Co-authored-by: Khristinin Nikita <[email protected]>

packages/kbn-securitysolution-io-ts-types/index.ts

@@ -25,6 +25,7 @@ export * from './src/non_empty_array';
 export * from './src/non_empty_or_nullable_string_array';
 export * from './src/non_empty_string_array';
 export * from './src/non_empty_string';
+export * from './src/number_between_zero_and_one_inclusive';
 export * from './src/only_false_allowed';
 export * from './src/operator';
 export * from './src/positive_integer_greater_than_zero';

packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.test.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+import { NumberBetweenZeroAndOneInclusive } from '.';
+import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils';
+
+describe('NumberBetweenZeroAndOneInclusive', () => {
+  test('it should validate 1', () => {
+    const payload = 1;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should validate a zero', () => {
+    const payload = 0;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should validate a float between 0 and 1', () => {
+    const payload = 0.58;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should NOT validate a negative number', () => {
+    const payload = -1;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "-1" supplied to "NumberBetweenZeroAndOneInclusive"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should NOT validate NaN', () => {
+    const payload = NaN;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "NaN" supplied to "NumberBetweenZeroAndOneInclusive"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should NOT validate Infinity', () => {
+    const payload = Infinity;
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "Infinity" supplied to "NumberBetweenZeroAndOneInclusive"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should NOT validate a string', () => {
+    const payload = 'some string';
+    const decoded = NumberBetweenZeroAndOneInclusive.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "some string" supplied to "NumberBetweenZeroAndOneInclusive"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+});

packages/kbn-securitysolution-io-ts-types/src/number_between_zero_and_one_inclusive/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+import { Either } from 'fp-ts/lib/Either';
+
+/**
+ * Types a number between 0 and 1 inclusive. Useful for specifying a probability, weighting, etc.
+ */
+export const NumberBetweenZeroAndOneInclusive = new t.Type<number, number, unknown>(
+  'NumberBetweenZeroAndOneInclusive',
+  t.number.is,
+  (input, context): Either<t.Errors, number> => {
+    return typeof input === 'number' &&
+      !Number.isNaN(input) &&
+      Number.isFinite(input) &&
+      input >= 0 &&
+      input <= 1
+      ? t.success(input)
+      : t.failure(input, context);
+  },
+  t.identity
+);

x-pack/plugins/security_solution/common/constants.ts

@@ -41,6 +41,7 @@ export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const;
 export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const;
 export const DEFAULT_LISTS_INDEX = '.lists' as const;
 export const DEFAULT_ITEMS_INDEX = '.items' as const;
+export const DEFAULT_RISK_SCORE_PAGE_SIZE = 1000 as const;
 // The DEFAULT_MAX_SIGNALS value exists also in `x-pack/plugins/cases/common/constants.ts`
 // If either changes, engineer should ensure both values are updated
 export const DEFAULT_MAX_SIGNALS = 100 as const;
@@ -314,6 +315,8 @@ export const RISK_SCORE_CREATE_INDEX = `${INTERNAL_RISK_SCORE_URL}/indices/creat
 export const RISK_SCORE_DELETE_INDICES = `${INTERNAL_RISK_SCORE_URL}/indices/delete`;
 export const RISK_SCORE_CREATE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/create`;
 export const RISK_SCORE_DELETE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/delete`;
+export const RISK_SCORE_PREVIEW_URL = `${INTERNAL_RISK_SCORE_URL}/preview`;
+
 /**
  * Internal detection engine routes
  */

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -116,6 +116,11 @@ export const allowedExperimentalValues = Object.freeze({
    * The flag doesn't have to be documented and has to be removed after the feature is ready to release.
    */
   detectionsCoverageOverview: false,
+
+  /**
+   * Enables experimental Entity Analytics HTTP endpoints
+   */
+  riskScoringRoutesEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/common/risk_engine/after_keys.test.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { pipe } from 'fp-ts/lib/pipeable';
+import { left } from 'fp-ts/lib/Either';
+import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils';
+
+import { afterKeysSchema } from './after_keys';
+
+describe('after_keys schema', () => {
+  it('allows an empty object', () => {
+    const payload = {};
+    const decoded = afterKeysSchema.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  it('allows a valid host key', () => {
+    const payload = { host: { 'host.name': 'hello' } };
+    const decoded = afterKeysSchema.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  it('allows a valid user key', () => {
+    const payload = { user: { 'user.name': 'hello' } };
+    const decoded = afterKeysSchema.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  it('allows both valid host and user keys', () => {
+    const payload = { user: { 'user.name': 'hello' }, host: { 'host.name': 'hello' } };
+    const decoded = afterKeysSchema.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  it('removes an unknown identifier key if used', () => {
+    const payload = { bad: 'key' };
+    const decoded = afterKeysSchema.decode(payload);
+    const message = pipe(decoded, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/security_solution/common/risk_engine/after_keys.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+
+const afterKeySchema = t.record(t.string, t.string);
+export type AfterKeySchema = t.TypeOf<typeof afterKeySchema>;
+export type AfterKey = AfterKeySchema;
+
+export const afterKeysSchema = t.exact(
+  t.partial({
+    host: afterKeySchema,
+    user: afterKeySchema,
+  })
+);
+export type AfterKeysSchema = t.TypeOf<typeof afterKeysSchema>;
+export type AfterKeys = AfterKeysSchema;

x-pack/plugins/security_solution/common/risk_engine/identifier_types.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+
+export const identifierTypeSchema = t.keyof({ user: null, host: null });
+export type IdentifierTypeSchema = t.TypeOf<typeof identifierTypeSchema>;
+export type IdentifierType = IdentifierTypeSchema;

x-pack/plugins/security_solution/common/risk_engine/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './after_keys';
+export * from './risk_weights';
+export * from './identifier_types';

x-pack/plugins/security_solution/common/risk_engine/risk_score_preview/request_schema.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { DataViewId } from '../../detection_engine/rule_schema';
+import { afterKeysSchema } from '../after_keys';
+import { identifierTypeSchema } from '../identifier_types';
+import { riskWeightsSchema } from '../risk_weights/schema';
+
+export const riskScorePreviewRequestSchema = t.exact(
+  t.partial({
+    after_keys: afterKeysSchema,
+    data_view_id: DataViewId,
+    debug: t.boolean,
+    filter: t.unknown,
+    page_size: t.number,
+    identifier_type: identifierTypeSchema,
+    range: t.type({
+      start: t.string,
+      end: t.string,
+    }),
+    weights: riskWeightsSchema,
+  })
+);
+export type RiskScorePreviewRequestSchema = t.TypeOf<typeof riskScorePreviewRequestSchema>;

x-pack/plugins/security_solution/common/risk_engine/risk_weights/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './types';
+export type { RiskWeight, RiskWeights, GlobalRiskWeight, RiskCategoryRiskWeight } from './schema';