Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/elastic/kibana into move-st…
Browse files Browse the repository at this point in the history
…ructure-rule_execution_test
  • Loading branch information
WafaaNasr committed Nov 16, 2023
2 parents 5434795 + db5176b commit fde47c6
Show file tree
Hide file tree
Showing 183 changed files with 6,671 additions and 2,408 deletions.
4 changes: 2 additions & 2 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ packages/analytics/shippers/gainsight @elastic/kibana-core
packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam
x-pack/plugins/apm_data_access @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team
x-pack/plugins/apm @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
packages/kbn-apm-utils @elastic/obs-ux-infra_services-team
test/plugin_functional/plugins/app_link_test @elastic/kibana-core
x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core
Expand Down
2 changes: 1 addition & 1 deletion config/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ xpack.alerting.rules.run.actions.max: 3000
xpack.alerting.rules.run.timeout: 1m
xpack.alerting.rules.run.ruleTypeOverrides:
- id: siem.indicatorRule
timeout: 1m
timeout: 10m
xpack.alerting.rules.minimumScheduleInterval.enforce: true
xpack.alerting.rules.maxScheduledPerMinute: 400
xpack.actions.run.maxAttempts: 10
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-apm-synthtrace-client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,4 @@ export { dedot } from './src/lib/utils/dedot';
export { generateLongId, generateShortId } from './src/lib/utils/generate_id';
export { appendHash, hashKeysOf } from './src/lib/utils/hash';
export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
export { log, type LogDocument } from './src/lib/logs';
2 changes: 1 addition & 1 deletion packages/kbn-apm-synthtrace-client/kibana.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"type": "shared-common",
"id": "@kbn/apm-synthtrace-client",
"devOnly": true,
"owner": "@elastic/obs-ux-infra_services-team"
"owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
}
77 changes: 77 additions & 0 deletions packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { Fields } from '../entity';
import { Serializable } from '../serializable';

export type LogDocument = Fields &
Partial<{
'input.type': string;
'log.file.path'?: string;
'service.name'?: string;
'data_stream.namespace': string;
'data_stream.type': string;
'data_stream.dataset': string;
message?: string;
'event.dataset': string;
'log.level'?: string;
'host.name'?: string;
'trace.id'?: string;
'agent.name'?: string;
'orchestrator.cluster.name'?: string;
'orchestrator.cluster.id'?: string;
'orchestrator.resource.id'?: string;
'cloud.provider'?: string;
'cloud.region'?: string;
'cloud.availability_zone'?: string;
'cloud.project.id'?: string;
'cloud.instance.id'?: string;
}>;

class Log extends Serializable<LogDocument> {
service(name: string) {
this.fields['service.name'] = name;
return this;
}

namespace(value: string) {
this.fields['data_stream.namespace'] = value;
return this;
}

dataset(value: string) {
this.fields['data_stream.dataset'] = value;
this.fields['event.dataset'] = value;
return this;
}

logLevel(level: string) {
this.fields['log.level'] = level;
return this;
}

message(message: string) {
this.fields.message = message;
return this;
}
}

function create(): Log {
return new Log({
'input.type': 'logs',
'data_stream.namespace': 'default',
'data_stream.type': 'logs',
'data_stream.dataset': 'synth',
'event.dataset': 'synth',
'host.name': 'synth-host',
});
}

export const log = {
create,
};
27 changes: 19 additions & 8 deletions packages/kbn-apm-synthtrace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ This library can currently be used in two ways:
- `Instance`: a single instance of a monitored service. E.g., the workload for a monitored service might be spread across multiple containers. An `Instance` object contains fields like `service.node.name` and `container.id`.
- `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets.
- `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html)
- `Log`: An instance of Log generating Service which supports additional helpers to customise fields like `messages`, `logLevel`

#### Example

Expand Down Expand Up @@ -109,12 +110,22 @@ node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost

The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**

### Understanding Scenario Files

Scenario files accept 3 arguments, 2 of them optional and 1 mandatory

| Arguments | Type | Description |
|-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------|
| `generate` | mandatory | This is the main function responsible for returning the events which will be indexed |
| `bootstrap` | optional | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with |
| `setClient` | optional | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned |

The following options are supported:

### Connection options

| Option | Type | Default | Description |
| ------------------- | -------- | :------ | ------------------------------------------------------------------------------------------ |
|---------------------|----------|:--------|--------------------------------------------------------------------------------------------|
| `--target` | [string] | | Elasticsearch target |
| `--kibana` | [string] | | Kibana target, used to bootstrap datastreams/mappings/templates/settings |
| `--versionOverride` | [string] | | String to be used for `observer.version`. Defauls to the version of the installed package. |
Expand All @@ -129,12 +140,11 @@ Note:
### Scenario options

| Option | Type | Default | Description |
| ---------------- | --------- | :------ | ------------------------------------ |
|------------------|-----------|:--------|--------------------------------------|
| `--from` | [date] | `now()` | The start of the time window |
| `--to` | [date] | | The end of the time window |
| `--live` | [boolean] | | Generate and index data continuously |
| `--scenarioOpts` | | | Raw options specific to the scenario |

Note:

- The default `--to` is `15m`.
Expand All @@ -143,11 +153,12 @@ Note:

### Setup options

| Option | Type | Default | Description |
| ------------ | --------- | :------ | --------------------------------------- |
| `--clean` | [boolean] | `false` | Clean APM data before indexing new data |
| `--workers` | [number] | | Amount of Node.js worker threads |
| `--logLevel` | [enum] | `info` | Log level |
| Option | Type | Default | Description |
|--------------|-----------|:--------|-------------------------------------------------------------------------|
| `--clean` | [boolean] | `false` | Clean APM data before indexing new data |
| `--workers` | [number] | | Amount of Node.js worker threads |
| `--logLevel` | [enum] | `info` | Log level |
| `--type` | [string] | `apm` | Type of data to be generated, `log` must be passed when generating logs |

## Testing

Expand Down
2 changes: 2 additions & 0 deletions packages/kbn-apm-synthtrace/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_

export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';

export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';

export {
addObserverVersionTransform,
deleteSummaryFieldTransform,
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-apm-synthtrace/kibana.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"type": "shared-server",
"id": "@kbn/apm-synthtrace",
"devOnly": true,
"owner": "@elastic/obs-ux-infra_services-team"
"owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
}
17 changes: 12 additions & 5 deletions packages/kbn-apm-synthtrace/src/cli/scenario.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,24 @@
* Side Public License, v 1.
*/

import { SynthtraceGenerator, Timerange } from '@kbn/apm-synthtrace-client';
import { Readable } from 'stream';
import { ApmSynthtraceEsClient } from '../lib/apm/client/apm_synthtrace_es_client';
import { Timerange } from '@kbn/apm-synthtrace-client';
import { Logger } from '../lib/utils/create_logger';
import { RunOptions } from './utils/parse_run_cli_flags';
import { ApmSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
import { ScenarioReturnType } from '../lib/utils/with_client';

type Generate<TFields> = (options: {
range: Timerange;
}) => SynthtraceGenerator<TFields> | Array<SynthtraceGenerator<TFields>> | Readable;
clients: {
apmEsClient: ApmSynthtraceEsClient;
logsEsClient: LogsSynthtraceEsClient;
};
}) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;

export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
bootstrap?: (options: { apmEsClient: ApmSynthtraceEsClient }) => Promise<void>;
bootstrap?: (options: {
apmEsClient: ApmSynthtraceEsClient;
logsEsClient: LogsSynthtraceEsClient;
}) => Promise<void>;
generate: Generate<TFields>;
}>;
13 changes: 11 additions & 2 deletions packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
*/

import { createLogger } from '../../lib/utils/create_logger';
import { getEsClient } from './get_es_client';
import { getApmEsClient } from './get_apm_es_client';
import { getLogsEsClient } from './get_logs_es_client';
import { getKibanaClient } from './get_kibana_client';
import { getServiceUrls } from './get_service_urls';
import { RunOptions } from './parse_run_cli_flags';
Expand All @@ -26,22 +27,30 @@ export async function bootstrap(runOptions: RunOptions) {

const version = runOptions.versionOverride || latestPackageVersion;

const apmEsClient = getEsClient({
const apmEsClient = getApmEsClient({
target: esUrl,
logger,
concurrency: runOptions.concurrency,
version,
});

const logsEsClient = getLogsEsClient({
target: esUrl,
logger,
concurrency: runOptions.concurrency,
});

await kibanaClient.installApmPackage(latestPackageVersion);

if (runOptions.clean) {
await apmEsClient.clean();
await logsEsClient.clean();
}

return {
logger,
apmEsClient,
logsEsClient,
version,
kibanaUrl,
esUrl,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
*/

import { Client } from '@elastic/elasticsearch';
import { ApmSynthtraceEsClient } from '../../lib/apm/client/apm_synthtrace_es_client';
import { ApmSynthtraceEsClient } from '../../..';
import { Logger } from '../../lib/utils/create_logger';
import { RunOptions } from './parse_run_cli_flags';

export function getEsClient({
export function getApmEsClient({
target,
logger,
version,
Expand Down
31 changes: 31 additions & 0 deletions packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { Client } from '@elastic/elasticsearch';
import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client';
import { Logger } from '../../lib/utils/create_logger';
import { RunOptions } from './parse_run_cli_flags';

export function getLogsEsClient({
target,
logger,
concurrency,
}: Pick<RunOptions, 'concurrency'> & {
target: string;
logger: Logger;
}) {
const client = new Client({
node: target,
});

return new LogsSynthtraceEsClient({
client,
logger,
concurrency,
});
}
Loading

0 comments on commit fde47c6

Please sign in to comment.