diff --git a/.eslintrc.js b/.eslintrc.js index 797b84522df3f..c604844089ef4 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -978,6 +978,7 @@ module.exports = { files: [ 'x-pack/plugins/observability_solution/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', 'src/plugins/ai_assistant_management/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', + 'x-pack/packages/observability/logs_overview/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', ], rules: { '@kbn/i18n/strings_should_be_translated_with_i18n': 'warn', diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9b3c46d065fe1..974a7d39f63b3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -652,6 +652,7 @@ x-pack/packages/observability/alerting_test_data @elastic/obs-ux-management-team x-pack/test/cases_api_integration/common/plugins/observability @elastic/response-ops x-pack/packages/observability/get_padded_alert_time_range_util @elastic/obs-ux-management-team x-pack/plugins/observability_solution/observability_logs_explorer @elastic/obs-ux-logs-team +x-pack/packages/observability/logs_overview @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/observability_onboarding/e2e @elastic/obs-ux-logs-team @elastic/obs-ux-onboarding-team x-pack/plugins/observability_solution/observability_onboarding @elastic/obs-ux-logs-team @elastic/obs-ux-onboarding-team x-pack/plugins/observability_solution/observability @elastic/obs-ux-management-team diff --git a/package.json b/package.json index 57b84f1c46dcb..58cd08773696f 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "@storybook/react-docgen-typescript-plugin": "1.0.6--canary.9.cd77847.0", "@types/react": "~18.2.0", "@types/react-dom": "~18.2.0", + "@xstate5/react/**/xstate": "^5.18.1", "globby/fast-glob": "^3.2.11" }, "dependencies": { @@ -687,6 +688,7 @@ "@kbn/observability-fixtures-plugin": "link:x-pack/test/cases_api_integration/common/plugins/observability", "@kbn/observability-get-padded-alert-time-range-util": "link:x-pack/packages/observability/get_padded_alert_time_range_util", "@kbn/observability-logs-explorer-plugin": "link:x-pack/plugins/observability_solution/observability_logs_explorer", + "@kbn/observability-logs-overview": "link:x-pack/packages/observability/logs_overview", "@kbn/observability-onboarding-plugin": "link:x-pack/plugins/observability_solution/observability_onboarding", "@kbn/observability-plugin": "link:x-pack/plugins/observability_solution/observability", "@kbn/observability-shared-plugin": "link:x-pack/plugins/observability_solution/observability_shared", @@ -1050,6 +1052,7 @@ "@turf/helpers": "6.0.1", "@turf/length": "^6.0.2", "@xstate/react": "^3.2.2", + "@xstate5/react": "npm:@xstate/react@^4.1.2", "adm-zip": "^0.5.9", "ai": "^2.2.33", "ajv": "^8.12.0", @@ -1283,6 +1286,7 @@ "whatwg-fetch": "^3.0.0", "xml2js": "^0.5.0", "xstate": "^4.38.2", + "xstate5": "npm:xstate@^5.18.1", "xterm": "^5.1.0", "yauzl": "^2.10.0", "yazl": "^2.5.1", @@ -1304,6 +1308,7 @@ "@babel/plugin-proposal-optional-chaining": "^7.21.0", "@babel/plugin-proposal-private-methods": "^7.18.6", "@babel/plugin-transform-class-properties": "^7.24.7", + "@babel/plugin-transform-logical-assignment-operators": "^7.24.7", "@babel/plugin-transform-numeric-separator": "^7.24.7", "@babel/plugin-transform-runtime": "^7.24.7", "@babel/preset-env": "^7.24.7", diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entity.ts index 4d522ef07ff0e..b26dbfc7ffb46 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/entity.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/entity.ts @@ -7,6 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +export type ObjectEntry = [keyof T, T[keyof T]]; + export type Fields | undefined = undefined> = { '@timestamp'?: number; } & (TMeta extends undefined ? {} : Partial<{ meta: TMeta }>); @@ -27,4 +29,14 @@ export class Entity { return this; } + + overrides(overrides: Partial) { + const overrideEntries = Object.entries(overrides) as Array>; + + overrideEntries.forEach(([fieldName, value]) => { + this.fields[fieldName] = value; + }); + + return this; + } } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/gaussian_events.ts b/packages/kbn-apm-synthtrace-client/src/lib/gaussian_events.ts new file mode 100644 index 0000000000000..4f1db28017d29 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/gaussian_events.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { castArray } from 'lodash'; +import { SynthtraceGenerator } from '../types'; +import { Fields } from './entity'; +import { Serializable } from './serializable'; + +export class GaussianEvents { + constructor( + private readonly from: Date, + private readonly to: Date, + private readonly mean: Date, + private readonly width: number, + private readonly totalPoints: number + ) {} + + *generator( + map: ( + timestamp: number, + index: number + ) => Serializable | Array> + ): SynthtraceGenerator { + if (this.totalPoints <= 0) { + return; + } + + const startTime = this.from.getTime(); + const endTime = this.to.getTime(); + const meanTime = this.mean.getTime(); + const densityInterval = 1 / (this.totalPoints - 1); + + for (let eventIndex = 0; eventIndex < this.totalPoints; eventIndex++) { + const quantile = eventIndex * densityInterval; + + const standardScore = Math.sqrt(2) * inverseError(2 * quantile - 1); + const timestamp = Math.round(meanTime + standardScore * this.width); + + if (timestamp >= startTime && timestamp <= endTime) { + yield* this.generateEvents(timestamp, eventIndex, map); + } + } + } + + private *generateEvents( + timestamp: number, + eventIndex: number, + map: ( + timestamp: number, + index: number + ) => Serializable | Array> + ): Generator> { + const events = castArray(map(timestamp, eventIndex)); + for (const event of events) { + yield event; + } + } +} + +function inverseError(x: number): number { + const a = 0.147; + const sign = x < 0 ? -1 : 1; + + const part1 = 2 / (Math.PI * a) + Math.log(1 - x * x) / 2; + const part2 = Math.log(1 - x * x) / a; + + return sign * Math.sqrt(Math.sqrt(part1 * part1 - part2) - part1); +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts index 198949b482be3..30550d64c4df8 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts @@ -27,7 +27,7 @@ interface HostDocument extends Fields { 'cloud.provider'?: string; } -class Host extends Entity { +export class Host extends Entity { cpu({ cpuTotalValue }: { cpuTotalValue?: number } = {}) { return new HostMetrics({ ...this.fields, @@ -175,3 +175,11 @@ export function host(name: string): Host { 'cloud.provider': 'gcp', }); } + +export function minimalHost(name: string): Host { + return new Host({ + 'agent.id': 'synthtrace', + 'host.hostname': name, + 'host.name': name, + }); +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/index.ts index 853a9549ce02c..2957605cffcd3 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/infra/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/infra/index.ts @@ -8,7 +8,7 @@ */ import { dockerContainer, DockerContainerMetricsDocument } from './docker_container'; -import { host, HostMetricsDocument } from './host'; +import { host, HostMetricsDocument, minimalHost } from './host'; import { k8sContainer, K8sContainerMetricsDocument } from './k8s_container'; import { pod, PodMetricsDocument } from './pod'; import { awsRds, AWSRdsMetricsDocument } from './aws/rds'; @@ -24,6 +24,7 @@ export type InfraDocument = export const infra = { host, + minimalHost, pod, dockerContainer, k8sContainer, diff --git a/packages/kbn-apm-synthtrace-client/src/lib/interval.ts b/packages/kbn-apm-synthtrace-client/src/lib/interval.ts index 1d56c42e1fe12..5a5ed3ab5fdbe 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/interval.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/interval.ts @@ -34,6 +34,10 @@ interface IntervalOptions { rate?: number; } +interface StepDetails { + stepMilliseconds: number; +} + export class Interval { private readonly intervalAmount: number; private readonly intervalUnit: unitOfTime.DurationConstructor; @@ -46,12 +50,16 @@ export class Interval { this._rate = options.rate || 1; } + private getIntervalMilliseconds(): number { + return moment.duration(this.intervalAmount, this.intervalUnit).asMilliseconds(); + } + private getTimestamps() { const from = this.options.from.getTime(); const to = this.options.to.getTime(); let time: number = from; - const diff = moment.duration(this.intervalAmount, this.intervalUnit).asMilliseconds(); + const diff = this.getIntervalMilliseconds(); const timestamps: number[] = []; @@ -68,15 +76,19 @@ export class Interval { *generator( map: ( timestamp: number, - index: number + index: number, + stepDetails: StepDetails ) => Serializable | Array> ): SynthtraceGenerator { const timestamps = this.getTimestamps(); + const stepDetails: StepDetails = { + stepMilliseconds: this.getIntervalMilliseconds(), + }; let index = 0; for (const timestamp of timestamps) { - const events = castArray(map(timestamp, index)); + const events = castArray(map(timestamp, index, stepDetails)); index++; for (const event of events) { yield event; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts index e19f0f6fd6565..2bbc59eb37e70 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts @@ -68,6 +68,7 @@ export type LogDocument = Fields & 'event.duration': number; 'event.start': Date; 'event.end': Date; + labels?: Record; test_field: string | string[]; date: Date; severity: string; @@ -156,6 +157,26 @@ function create(logsOptions: LogsOptions = defaultLogsOptions): Log { ).dataset('synth'); } +function createMinimal({ + dataset = 'synth', + namespace = 'default', +}: { + dataset?: string; + namespace?: string; +} = {}): Log { + return new Log( + { + 'input.type': 'logs', + 'data_stream.namespace': namespace, + 'data_stream.type': 'logs', + 'data_stream.dataset': dataset, + 'event.dataset': dataset, + }, + { isLogsDb: false } + ); +} + export const log = { create, + createMinimal, }; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.test.ts b/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.test.ts new file mode 100644 index 0000000000000..0741884550f32 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { PoissonEvents } from './poisson_events'; +import { Serializable } from './serializable'; + +describe('poisson events', () => { + it('generates events within the given time range', () => { + const poissonEvents = new PoissonEvents(new Date(1000), new Date(2000), 10); + + const events = Array.from( + poissonEvents.generator((timestamp) => new Serializable({ '@timestamp': timestamp })) + ); + + expect(events.length).toBeGreaterThanOrEqual(1); + + for (const event of events) { + expect(event.fields['@timestamp']).toBeGreaterThanOrEqual(1000); + expect(event.fields['@timestamp']).toBeLessThanOrEqual(2000); + } + }); + + it('generates at least one event if the rate is greater than 0', () => { + const poissonEvents = new PoissonEvents(new Date(1000), new Date(2000), 1); + + const events = Array.from( + poissonEvents.generator((timestamp) => new Serializable({ '@timestamp': timestamp })) + ); + + expect(events.length).toBeGreaterThanOrEqual(1); + + for (const event of events) { + expect(event.fields['@timestamp']).toBeGreaterThanOrEqual(1000); + expect(event.fields['@timestamp']).toBeLessThanOrEqual(2000); + } + }); + + it('generates no event if the rate is 0', () => { + const poissonEvents = new PoissonEvents(new Date(1000), new Date(2000), 0); + + const events = Array.from( + poissonEvents.generator((timestamp) => new Serializable({ '@timestamp': timestamp })) + ); + + expect(events.length).toBe(0); + }); +}); diff --git a/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.ts b/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.ts new file mode 100644 index 0000000000000..e7fd24b8323e7 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/poisson_events.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { castArray } from 'lodash'; +import { SynthtraceGenerator } from '../types'; +import { Fields } from './entity'; +import { Serializable } from './serializable'; + +export class PoissonEvents { + constructor( + private readonly from: Date, + private readonly to: Date, + private readonly rate: number + ) {} + + private getTotalTimePeriod(): number { + return this.to.getTime() - this.from.getTime(); + } + + private getInterarrivalTime(): number { + const distribution = -Math.log(1 - Math.random()) / this.rate; + const totalTimePeriod = this.getTotalTimePeriod(); + return Math.floor(distribution * totalTimePeriod); + } + + *generator( + map: ( + timestamp: number, + index: number + ) => Serializable | Array> + ): SynthtraceGenerator { + if (this.rate <= 0) { + return; + } + + let currentTime = this.from.getTime(); + const endTime = this.to.getTime(); + let eventIndex = 0; + + while (currentTime < endTime) { + const interarrivalTime = this.getInterarrivalTime(); + currentTime += interarrivalTime; + + if (currentTime < endTime) { + yield* this.generateEvents(currentTime, eventIndex, map); + eventIndex++; + } + } + + // ensure at least one event has been emitted + if (this.rate > 0 && eventIndex === 0) { + const forcedEventTime = + this.from.getTime() + Math.floor(Math.random() * this.getTotalTimePeriod()); + yield* this.generateEvents(forcedEventTime, eventIndex, map); + } + } + + private *generateEvents( + timestamp: number, + eventIndex: number, + map: ( + timestamp: number, + index: number + ) => Serializable | Array> + ): Generator> { + const events = castArray(map(timestamp, eventIndex)); + for (const event of events) { + yield event; + } + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/timerange.ts b/packages/kbn-apm-synthtrace-client/src/lib/timerange.ts index ccdea4ee75197..1c6f12414a148 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/timerange.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/timerange.ts @@ -9,10 +9,12 @@ import datemath from '@kbn/datemath'; import type { Moment } from 'moment'; +import { GaussianEvents } from './gaussian_events'; import { Interval } from './interval'; +import { PoissonEvents } from './poisson_events'; export class Timerange { - constructor(private from: Date, private to: Date) {} + constructor(public readonly from: Date, public readonly to: Date) {} interval(interval: string) { return new Interval({ from: this.from, to: this.to, interval }); @@ -21,6 +23,29 @@ export class Timerange { ratePerMinute(rate: number) { return this.interval(`1m`).rate(rate); } + + poissonEvents(rate: number) { + return new PoissonEvents(this.from, this.to, rate); + } + + gaussianEvents(mean: Date, width: number, totalPoints: number) { + return new GaussianEvents(this.from, this.to, mean, width, totalPoints); + } + + splitInto(segmentCount: number): Timerange[] { + const duration = this.to.getTime() - this.from.getTime(); + const segmentDuration = duration / segmentCount; + + return Array.from({ length: segmentCount }, (_, i) => { + const from = new Date(this.from.getTime() + i * segmentDuration); + const to = new Date(from.getTime() + segmentDuration); + return new Timerange(from, to); + }); + } + + toString() { + return `Timerange(from=${this.from.toISOString()}, to=${this.to.toISOString()})`; + } } type DateLike = Date | number | Moment | string; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/distributed_unstructured_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/distributed_unstructured_logs.ts new file mode 100644 index 0000000000000..83860635ae64a --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/distributed_unstructured_logs.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { infra, LogDocument, log } from '@kbn/apm-synthtrace-client'; +import { fakerEN as faker } from '@faker-js/faker'; +import { z } from '@kbn/zod'; +import { Scenario } from '../cli/scenario'; +import { withClient } from '../lib/utils/with_client'; +import { + LogMessageGenerator, + generateUnstructuredLogMessage, + unstructuredLogMessageGenerators, +} from './helpers/unstructured_logs'; + +const scenarioOptsSchema = z.intersection( + z.object({ + randomSeed: z.number().default(0), + messageGroup: z + .enum([ + 'httpAccess', + 'userAuthentication', + 'networkEvent', + 'dbOperations', + 'taskOperations', + 'degradedOperations', + 'errorOperations', + ]) + .default('dbOperations'), + }), + z + .discriminatedUnion('distribution', [ + z.object({ + distribution: z.literal('uniform'), + rate: z.number().default(1), + }), + z.object({ + distribution: z.literal('poisson'), + rate: z.number().default(1), + }), + z.object({ + distribution: z.literal('gaussian'), + mean: z.coerce.date().describe('Time of the peak of the gaussian distribution'), + width: z.number().default(5000).describe('Width of the gaussian distribution in ms'), + totalPoints: z + .number() + .default(100) + .describe('Total number of points in the gaussian distribution'), + }), + ]) + .default({ distribution: 'uniform', rate: 1 }) +); + +type ScenarioOpts = z.output; + +const scenario: Scenario = async (runOptions) => { + return { + generate: ({ range, clients: { logsEsClient } }) => { + const { logger } = runOptions; + const scenarioOpts = scenarioOptsSchema.parse(runOptions.scenarioOpts ?? {}); + + faker.seed(scenarioOpts.randomSeed); + faker.setDefaultRefDate(range.from.toISOString()); + + logger.debug(`Generating ${scenarioOpts.distribution} logs...`); + + // Logs Data logic + const LOG_LEVELS = ['info', 'debug', 'error', 'warn', 'trace', 'fatal']; + + const clusterDefinions = [ + { + 'orchestrator.cluster.id': faker.string.nanoid(), + 'orchestrator.cluster.name': 'synth-cluster-1', + 'orchestrator.namespace': 'default', + 'cloud.provider': 'gcp', + 'cloud.region': 'eu-central-1', + 'cloud.availability_zone': 'eu-central-1a', + 'cloud.project.id': faker.string.nanoid(), + }, + { + 'orchestrator.cluster.id': faker.string.nanoid(), + 'orchestrator.cluster.name': 'synth-cluster-2', + 'orchestrator.namespace': 'production', + 'cloud.provider': 'aws', + 'cloud.region': 'us-east-1', + 'cloud.availability_zone': 'us-east-1a', + 'cloud.project.id': faker.string.nanoid(), + }, + { + 'orchestrator.cluster.id': faker.string.nanoid(), + 'orchestrator.cluster.name': 'synth-cluster-3', + 'orchestrator.namespace': 'kube', + 'cloud.provider': 'azure', + 'cloud.region': 'area-51', + 'cloud.availability_zone': 'area-51a', + 'cloud.project.id': faker.string.nanoid(), + }, + ]; + + const hostEntities = [ + { + 'host.name': 'host-1', + 'agent.id': 'synth-agent-1', + 'agent.name': 'nodejs', + 'cloud.instance.id': faker.string.nanoid(), + 'orchestrator.resource.id': faker.string.nanoid(), + ...clusterDefinions[0], + }, + { + 'host.name': 'host-2', + 'agent.id': 'synth-agent-2', + 'agent.name': 'custom', + 'cloud.instance.id': faker.string.nanoid(), + 'orchestrator.resource.id': faker.string.nanoid(), + ...clusterDefinions[1], + }, + { + 'host.name': 'host-3', + 'agent.id': 'synth-agent-3', + 'agent.name': 'python', + 'cloud.instance.id': faker.string.nanoid(), + 'orchestrator.resource.id': faker.string.nanoid(), + ...clusterDefinions[2], + }, + ].map((hostDefinition) => + infra.minimalHost(hostDefinition['host.name']).overrides(hostDefinition) + ); + + const serviceNames = Array(3) + .fill(null) + .map((_, idx) => `synth-service-${idx}`); + + const generatorFactory = + scenarioOpts.distribution === 'uniform' + ? range.interval('1s').rate(scenarioOpts.rate) + : scenarioOpts.distribution === 'poisson' + ? range.poissonEvents(scenarioOpts.rate) + : range.gaussianEvents(scenarioOpts.mean, scenarioOpts.width, scenarioOpts.totalPoints); + + const logs = generatorFactory.generator((timestamp) => { + const entity = faker.helpers.arrayElement(hostEntities); + const serviceName = faker.helpers.arrayElement(serviceNames); + const level = faker.helpers.arrayElement(LOG_LEVELS); + const messages = logMessageGenerators[scenarioOpts.messageGroup](faker); + + return messages.map((message) => + log + .createMinimal() + .message(message) + .logLevel(level) + .service(serviceName) + .overrides({ + ...entity.fields, + labels: { + scenario: 'rare', + population: scenarioOpts.distribution, + }, + }) + .timestamp(timestamp) + ); + }); + + return [ + withClient( + logsEsClient, + logger.perf('generating_logs', () => [logs]) + ), + ]; + }, + }; +}; + +export default scenario; + +const logMessageGenerators = { + httpAccess: generateUnstructuredLogMessage([unstructuredLogMessageGenerators.httpAccess]), + userAuthentication: generateUnstructuredLogMessage([ + unstructuredLogMessageGenerators.userAuthentication, + ]), + networkEvent: generateUnstructuredLogMessage([unstructuredLogMessageGenerators.networkEvent]), + dbOperations: generateUnstructuredLogMessage([unstructuredLogMessageGenerators.dbOperation]), + taskOperations: generateUnstructuredLogMessage([ + unstructuredLogMessageGenerators.taskStatusSuccess, + ]), + degradedOperations: generateUnstructuredLogMessage([ + unstructuredLogMessageGenerators.taskStatusFailure, + ]), + errorOperations: generateUnstructuredLogMessage([ + unstructuredLogMessageGenerators.error, + unstructuredLogMessageGenerators.restart, + ]), +} satisfies Record; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/helpers/unstructured_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/helpers/unstructured_logs.ts new file mode 100644 index 0000000000000..490bd449e2b60 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/helpers/unstructured_logs.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Faker, faker } from '@faker-js/faker'; + +export type LogMessageGenerator = (f: Faker) => string[]; + +export const unstructuredLogMessageGenerators = { + httpAccess: (f: Faker) => [ + `${f.internet.ip()} - - [${f.date + .past() + .toISOString() + .replace('T', ' ') + .replace( + /\..+/, + '' + )}] "${f.internet.httpMethod()} ${f.internet.url()} HTTP/1.1" ${f.helpers.arrayElement([ + 200, 301, 404, 500, + ])} ${f.number.int({ min: 100, max: 5000 })}`, + ], + dbOperation: (f: Faker) => [ + `${f.database.engine()}: ${f.database.column()} ${f.helpers.arrayElement([ + 'created', + 'updated', + 'deleted', + 'inserted', + ])} successfully ${f.number.int({ max: 100000 })} times`, + ], + taskStatusSuccess: (f: Faker) => [ + `${f.hacker.noun()}: ${f.word.words()} ${f.helpers.arrayElement([ + 'triggered', + 'executed', + 'processed', + 'handled', + ])} successfully at ${f.date.recent().toISOString()}`, + ], + taskStatusFailure: (f: Faker) => [ + `${f.hacker.noun()}: ${f.helpers.arrayElement([ + 'triggering', + 'execution', + 'processing', + 'handling', + ])} of ${f.word.words()} failed at ${f.date.recent().toISOString()}`, + ], + error: (f: Faker) => [ + `${f.helpers.arrayElement([ + 'Error', + 'Exception', + 'Failure', + 'Crash', + 'Bug', + 'Issue', + ])}: ${f.hacker.phrase()}`, + `Stopping ${f.number.int(42)} background tasks...`, + 'Shutting down process...', + ], + restart: (f: Faker) => { + const service = f.database.engine(); + return [ + `Restarting ${service}...`, + `Waiting for queue to drain...`, + `Service ${service} restarted ${f.helpers.arrayElement([ + 'successfully', + 'with errors', + 'with warnings', + ])}`, + ]; + }, + userAuthentication: (f: Faker) => [ + `User ${f.internet.userName()} ${f.helpers.arrayElement([ + 'logged in', + 'logged out', + 'failed to login', + ])}`, + ], + networkEvent: (f: Faker) => [ + `Network ${f.helpers.arrayElement([ + 'connection', + 'disconnection', + 'data transfer', + ])} ${f.helpers.arrayElement(['from', 'to'])} ${f.internet.ip()}`, + ], +} satisfies Record; + +export const generateUnstructuredLogMessage = + (generators: LogMessageGenerator[] = Object.values(unstructuredLogMessageGenerators)) => + (f: Faker = faker) => + f.helpers.arrayElement(generators)(f); diff --git a/packages/kbn-apm-synthtrace/tsconfig.json b/packages/kbn-apm-synthtrace/tsconfig.json index d0f5c5801597a..db93e36421b83 100644 --- a/packages/kbn-apm-synthtrace/tsconfig.json +++ b/packages/kbn-apm-synthtrace/tsconfig.json @@ -10,6 +10,7 @@ "@kbn/apm-synthtrace-client", "@kbn/dev-utils", "@kbn/elastic-agent-utils", + "@kbn/zod", ], "exclude": [ "target/**/*", diff --git a/packages/kbn-management/settings/setting_ids/index.ts b/packages/kbn-management/settings/setting_ids/index.ts index 2b8c5de0b71df..e926007f77f25 100644 --- a/packages/kbn-management/settings/setting_ids/index.ts +++ b/packages/kbn-management/settings/setting_ids/index.ts @@ -142,6 +142,7 @@ export const OBSERVABILITY_APM_ENABLE_SERVICE_INVENTORY_TABLE_SEARCH_BAR = 'observability:apmEnableServiceInventoryTableSearchBar'; export const OBSERVABILITY_LOGS_EXPLORER_ALLOWED_DATA_VIEWS_ID = 'observability:logsExplorer:allowedDataViews'; +export const OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID = 'observability:newLogsOverview'; export const OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE = 'observability:entityCentricExperience'; export const OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID = 'observability:logSources'; export const OBSERVABILITY_ENABLE_LOGS_STREAM = 'observability:enableLogsStream'; diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 539d3098030e0..52a837724480d 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -247,6 +247,18 @@ export function getWebpackConfig( }, }, }, + { + test: /node_modules\/@?xstate5\/.*\.js$/, + use: { + loader: 'babel-loader', + options: { + babelrc: false, + envName: worker.dist ? 'production' : 'development', + presets: [BABEL_PRESET], + plugins: ['@babel/plugin-transform-logical-assignment-operators'], + }, + }, + }, { test: /\.(html|md|txt|tmpl)$/, use: { diff --git a/packages/kbn-storybook/src/webpack.config.ts b/packages/kbn-storybook/src/webpack.config.ts index fb901692e7f66..b03d78dbbc190 100644 --- a/packages/kbn-storybook/src/webpack.config.ts +++ b/packages/kbn-storybook/src/webpack.config.ts @@ -125,6 +125,17 @@ export default ({ config: storybookConfig }: { config: Configuration }) => { }, ], }, + { + test: /node_modules\/@?xstate5\/.*\.js$/, + use: { + loader: 'babel-loader', + options: { + babelrc: false, + presets: [require.resolve('@kbn/babel-preset/webpack_preset')], + plugins: ['@babel/plugin-transform-logical-assignment-operators'], + }, + }, + }, ], }, plugins: [new IgnoreNotFoundExportPlugin()], diff --git a/packages/kbn-xstate-utils/kibana.jsonc b/packages/kbn-xstate-utils/kibana.jsonc index cd1151a3f2103..1fb3507854b98 100644 --- a/packages/kbn-xstate-utils/kibana.jsonc +++ b/packages/kbn-xstate-utils/kibana.jsonc @@ -1,5 +1,5 @@ { - "type": "shared-common", + "type": "shared-browser", "id": "@kbn/xstate-utils", "owner": "@elastic/obs-ux-logs-team" } diff --git a/packages/kbn-xstate-utils/src/console_inspector.ts b/packages/kbn-xstate-utils/src/console_inspector.ts new file mode 100644 index 0000000000000..8792ab44f3c28 --- /dev/null +++ b/packages/kbn-xstate-utils/src/console_inspector.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { + ActorRefLike, + AnyActorRef, + InspectedActorEvent, + InspectedEventEvent, + InspectedSnapshotEvent, + InspectionEvent, +} from 'xstate5'; +import { isDevMode } from './dev_tools'; + +export const createConsoleInspector = () => { + if (!isDevMode()) { + return () => {}; + } + + // eslint-disable-next-line no-console + const log = console.info.bind(console); + + const logActorEvent = (actorEvent: InspectedActorEvent) => { + if (isActorRef(actorEvent.actorRef)) { + log( + '✨ %c%s%c is a new actor of type %c%s%c:', + ...styleAsActor(actorEvent.actorRef.id), + ...styleAsKeyword(actorEvent.type), + actorEvent.actorRef + ); + } else { + log('✨ New %c%s%c actor without id:', ...styleAsKeyword(actorEvent.type), actorEvent); + } + }; + + const logEventEvent = (eventEvent: InspectedEventEvent) => { + if (isActorRef(eventEvent.actorRef)) { + log( + '🔔 %c%s%c received event %c%s%c from %c%s%c:', + ...styleAsActor(eventEvent.actorRef.id), + ...styleAsKeyword(eventEvent.event.type), + ...styleAsKeyword(eventEvent.sourceRef?.id), + eventEvent + ); + } else { + log('🔔 Event', ...styleAsKeyword(eventEvent.event.type), ':', eventEvent); + } + }; + + const logSnapshotEvent = (snapshotEvent: InspectedSnapshotEvent) => { + if (isActorRef(snapshotEvent.actorRef)) { + log( + '📸 %c%s%c updated due to %c%s%c:', + ...styleAsActor(snapshotEvent.actorRef.id), + ...styleAsKeyword(snapshotEvent.event.type), + snapshotEvent.snapshot + ); + } else { + log('📸 Snapshot due to %c%s%c:', ...styleAsKeyword(snapshotEvent.event.type), snapshotEvent); + } + }; + + return (inspectionEvent: InspectionEvent) => { + if (inspectionEvent.type === '@xstate.actor') { + logActorEvent(inspectionEvent); + } else if (inspectionEvent.type === '@xstate.event') { + logEventEvent(inspectionEvent); + } else if (inspectionEvent.type === '@xstate.snapshot') { + logSnapshotEvent(inspectionEvent); + } else { + log(`❓ Received inspection event:`, inspectionEvent); + } + }; +}; + +const isActorRef = (actorRefLike: ActorRefLike): actorRefLike is AnyActorRef => + 'id' in actorRefLike; + +const keywordStyle = 'font-weight: bold'; +const styleAsKeyword = (value: any) => [keywordStyle, value, ''] as const; + +const actorStyle = 'font-weight: bold; text-decoration: underline'; +const styleAsActor = (value: any) => [actorStyle, value, ''] as const; diff --git a/packages/kbn-xstate-utils/src/index.ts b/packages/kbn-xstate-utils/src/index.ts index 107585ba2096f..3edf83e8a32c2 100644 --- a/packages/kbn-xstate-utils/src/index.ts +++ b/packages/kbn-xstate-utils/src/index.ts @@ -9,5 +9,6 @@ export * from './actions'; export * from './dev_tools'; +export * from './console_inspector'; export * from './notification_channel'; export * from './types'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index dc2d2ad2c5de2..e5ddfbe4dd037 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -705,4 +705,10 @@ export const stackManagementSchema: MakeSchemaFrom = { _meta: { description: 'Non-default value of setting.' }, }, }, + 'observability:newLogsOverview': { + type: 'boolean', + _meta: { + description: 'Enable the new logs overview component.', + }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index ef20ab223dfb6..2acb487e7ed08 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -56,6 +56,7 @@ export interface UsageStats { 'observability:logsExplorer:allowedDataViews': string[]; 'observability:logSources': string[]; 'observability:enableLogsStream': boolean; + 'observability:newLogsOverview': boolean; 'observability:aiAssistantSimulatedFunctionCalling': boolean; 'observability:aiAssistantSearchConnectorIndexPattern': string; 'visualization:heatmap:maxBuckets': number; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 958280d9eba00..830cffc17cf1c 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -10768,6 +10768,12 @@ "description": "Non-default value of setting." } }, + "observability:newLogsOverview": { + "type": "boolean", + "_meta": { + "description": "Enable the new logs overview component." + } + }, "observability:searchExcludedDataTiers": { "type": "array", "items": { diff --git a/tsconfig.base.json b/tsconfig.base.json index 3df30d9cf8c30..4bc68d806f043 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1298,6 +1298,8 @@ "@kbn/observability-get-padded-alert-time-range-util/*": ["x-pack/packages/observability/get_padded_alert_time_range_util/*"], "@kbn/observability-logs-explorer-plugin": ["x-pack/plugins/observability_solution/observability_logs_explorer"], "@kbn/observability-logs-explorer-plugin/*": ["x-pack/plugins/observability_solution/observability_logs_explorer/*"], + "@kbn/observability-logs-overview": ["x-pack/packages/observability/logs_overview"], + "@kbn/observability-logs-overview/*": ["x-pack/packages/observability/logs_overview/*"], "@kbn/observability-onboarding-e2e": ["x-pack/plugins/observability_solution/observability_onboarding/e2e"], "@kbn/observability-onboarding-e2e/*": ["x-pack/plugins/observability_solution/observability_onboarding/e2e/*"], "@kbn/observability-onboarding-plugin": ["x-pack/plugins/observability_solution/observability_onboarding"], diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index a46e291093411..50f2b77b84ad7 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -95,6 +95,9 @@ "xpack.observabilityLogsExplorer": "plugins/observability_solution/observability_logs_explorer", "xpack.observability_onboarding": "plugins/observability_solution/observability_onboarding", "xpack.observabilityShared": "plugins/observability_solution/observability_shared", + "xpack.observabilityLogsOverview": [ + "packages/observability/logs_overview/src/components" + ], "xpack.osquery": ["plugins/osquery"], "xpack.painlessLab": "plugins/painless_lab", "xpack.profiling": ["plugins/observability_solution/profiling"], diff --git a/x-pack/packages/observability/logs_overview/README.md b/x-pack/packages/observability/logs_overview/README.md new file mode 100644 index 0000000000000..20d3f0f02b7df --- /dev/null +++ b/x-pack/packages/observability/logs_overview/README.md @@ -0,0 +1,3 @@ +# @kbn/observability-logs-overview + +Empty package generated by @kbn/generate diff --git a/x-pack/packages/observability/logs_overview/index.ts b/x-pack/packages/observability/logs_overview/index.ts new file mode 100644 index 0000000000000..057d1d3acd152 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + LogsOverview, + LogsOverviewErrorContent, + LogsOverviewLoadingContent, + type LogsOverviewDependencies, + type LogsOverviewErrorContentProps, + type LogsOverviewProps, +} from './src/components/logs_overview'; +export type { + DataViewLogsSourceConfiguration, + IndexNameLogsSourceConfiguration, + LogsSourceConfiguration, + SharedSettingLogsSourceConfiguration, +} from './src/utils/logs_source'; diff --git a/x-pack/packages/observability/logs_overview/jest.config.js b/x-pack/packages/observability/logs_overview/jest.config.js new file mode 100644 index 0000000000000..2ee88ee990253 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/packages/observability/logs_overview'], +}; diff --git a/x-pack/packages/observability/logs_overview/kibana.jsonc b/x-pack/packages/observability/logs_overview/kibana.jsonc new file mode 100644 index 0000000000000..90b3375086720 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-browser", + "id": "@kbn/observability-logs-overview", + "owner": "@elastic/obs-ux-logs-team" +} diff --git a/x-pack/packages/observability/logs_overview/package.json b/x-pack/packages/observability/logs_overview/package.json new file mode 100644 index 0000000000000..77a529e7e59f7 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/observability-logs-overview", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0", + "sideEffects": false +} diff --git a/x-pack/packages/observability/logs_overview/src/components/discover_link/discover_link.tsx b/x-pack/packages/observability/logs_overview/src/components/discover_link/discover_link.tsx new file mode 100644 index 0000000000000..fe108289985a9 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/discover_link/discover_link.tsx @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { EuiButton } from '@elastic/eui'; +import type { DiscoverAppLocatorParams } from '@kbn/discover-plugin/common'; +import { FilterStateStore, buildCustomFilter } from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import { getRouterLinkProps } from '@kbn/router-utils'; +import type { SharePluginStart } from '@kbn/share-plugin/public'; +import React, { useCallback, useMemo } from 'react'; +import type { IndexNameLogsSourceConfiguration } from '../../utils/logs_source'; + +export interface DiscoverLinkProps { + documentFilters?: QueryDslQueryContainer[]; + logsSource: IndexNameLogsSourceConfiguration; + timeRange: { + start: string; + end: string; + }; + dependencies: DiscoverLinkDependencies; +} + +export interface DiscoverLinkDependencies { + share: SharePluginStart; +} + +export const DiscoverLink = React.memo( + ({ dependencies: { share }, documentFilters, logsSource, timeRange }: DiscoverLinkProps) => { + const discoverLocatorParams = useMemo( + () => ({ + dataViewSpec: { + id: logsSource.indexName, + name: logsSource.indexName, + title: logsSource.indexName, + timeFieldName: logsSource.timestampField, + }, + timeRange: { + from: timeRange.start, + to: timeRange.end, + }, + filters: documentFilters?.map((filter) => + buildCustomFilter( + logsSource.indexName, + filter, + false, + false, + categorizedLogsFilterLabel, + FilterStateStore.APP_STATE + ) + ), + }), + [ + documentFilters, + logsSource.indexName, + logsSource.timestampField, + timeRange.end, + timeRange.start, + ] + ); + + const discoverLocator = useMemo( + () => share.url.locators.get('DISCOVER_APP_LOCATOR'), + [share.url.locators] + ); + + const discoverUrl = useMemo( + () => discoverLocator?.getRedirectUrl(discoverLocatorParams), + [discoverLocatorParams, discoverLocator] + ); + + const navigateToDiscover = useCallback(() => { + discoverLocator?.navigate(discoverLocatorParams); + }, [discoverLocatorParams, discoverLocator]); + + const discoverLinkProps = getRouterLinkProps({ + href: discoverUrl, + onClick: navigateToDiscover, + }); + + return ( + + {discoverLinkTitle} + + ); + } +); + +export const discoverLinkTitle = i18n.translate( + 'xpack.observabilityLogsOverview.discoverLinkTitle', + { + defaultMessage: 'Open in Discover', + } +); + +export const categorizedLogsFilterLabel = i18n.translate( + 'xpack.observabilityLogsOverview.categorizedLogsFilterLabel', + { + defaultMessage: 'Categorized log entries', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/discover_link/index.ts b/x-pack/packages/observability/logs_overview/src/components/discover_link/index.ts new file mode 100644 index 0000000000000..738bf51d4529d --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/discover_link/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './discover_link'; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/index.ts b/x-pack/packages/observability/logs_overview/src/components/log_categories/index.ts new file mode 100644 index 0000000000000..786475396237c --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './log_categories'; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories.tsx new file mode 100644 index 0000000000000..6204667827281 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories.tsx @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { ISearchGeneric } from '@kbn/search-types'; +import { createConsoleInspector } from '@kbn/xstate-utils'; +import { useMachine } from '@xstate5/react'; +import React, { useCallback } from 'react'; +import { + categorizeLogsService, + createCategorizeLogsServiceImplementations, +} from '../../services/categorize_logs_service'; +import { IndexNameLogsSourceConfiguration } from '../../utils/logs_source'; +import { LogCategoriesErrorContent } from './log_categories_error_content'; +import { LogCategoriesLoadingContent } from './log_categories_loading_content'; +import { + LogCategoriesResultContent, + LogCategoriesResultContentDependencies, +} from './log_categories_result_content'; + +export interface LogCategoriesProps { + dependencies: LogCategoriesDependencies; + documentFilters?: QueryDslQueryContainer[]; + logsSource: IndexNameLogsSourceConfiguration; + // The time range could be made optional if we want to support an internal + // time range picker + timeRange: { + start: string; + end: string; + }; +} + +export type LogCategoriesDependencies = LogCategoriesResultContentDependencies & { + search: ISearchGeneric; +}; + +export const LogCategories: React.FC = ({ + dependencies, + documentFilters = [], + logsSource, + timeRange, +}) => { + const [categorizeLogsServiceState, sendToCategorizeLogsService] = useMachine( + categorizeLogsService.provide( + createCategorizeLogsServiceImplementations({ search: dependencies.search }) + ), + { + inspect: consoleInspector, + input: { + index: logsSource.indexName, + startTimestamp: timeRange.start, + endTimestamp: timeRange.end, + timeField: logsSource.timestampField, + messageField: logsSource.messageField, + documentFilters, + }, + } + ); + + const cancelOperation = useCallback(() => { + sendToCategorizeLogsService({ + type: 'cancel', + }); + }, [sendToCategorizeLogsService]); + + if (categorizeLogsServiceState.matches('done')) { + return ( + + ); + } else if (categorizeLogsServiceState.matches('failed')) { + return ; + } else if (categorizeLogsServiceState.matches('countingDocuments')) { + return ; + } else if ( + categorizeLogsServiceState.matches('fetchingSampledCategories') || + categorizeLogsServiceState.matches('fetchingRemainingCategories') + ) { + return ; + } else { + return null; + } +}; + +const consoleInspector = createConsoleInspector(); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_control_bar.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_control_bar.tsx new file mode 100644 index 0000000000000..4538b0ec2fd5d --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_control_bar.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import type { SharePluginStart } from '@kbn/share-plugin/public'; +import React from 'react'; +import type { IndexNameLogsSourceConfiguration } from '../../utils/logs_source'; +import { DiscoverLink } from '../discover_link'; + +export interface LogCategoriesControlBarProps { + documentFilters?: QueryDslQueryContainer[]; + logsSource: IndexNameLogsSourceConfiguration; + timeRange: { + start: string; + end: string; + }; + dependencies: LogCategoriesControlBarDependencies; +} + +export interface LogCategoriesControlBarDependencies { + share: SharePluginStart; +} + +export const LogCategoriesControlBar: React.FC = React.memo( + ({ dependencies, documentFilters, logsSource, timeRange }) => { + return ( + + + + + + ); + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_error_content.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_error_content.tsx new file mode 100644 index 0000000000000..1a335e3265294 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_error_content.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiCodeBlock, EuiEmptyPrompt } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export interface LogCategoriesErrorContentProps { + error?: Error; +} + +export const LogCategoriesErrorContent: React.FC = ({ error }) => { + return ( + {logsOverviewErrorTitle}} + body={ + +

{error?.stack ?? error?.toString() ?? unknownErrorDescription}

+ + } + layout="vertical" + /> + ); +}; + +const logsOverviewErrorTitle = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.errorTitle', + { + defaultMessage: 'Failed to categorize logs', + } +); + +const unknownErrorDescription = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.unknownErrorDescription', + { + defaultMessage: 'An unspecified error occurred.', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid.tsx new file mode 100644 index 0000000000000..d9e960685de99 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid.tsx @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiDataGrid, + EuiDataGridColumnSortingConfig, + EuiDataGridPaginationProps, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { createConsoleInspector } from '@kbn/xstate-utils'; +import { useMachine } from '@xstate5/react'; +import _ from 'lodash'; +import React, { useMemo } from 'react'; +import { assign, setup } from 'xstate5'; +import { LogCategory } from '../../types'; +import { + LogCategoriesGridCellDependencies, + LogCategoriesGridColumnId, + createCellContext, + logCategoriesGridColumnIds, + logCategoriesGridColumns, + renderLogCategoriesGridCell, +} from './log_categories_grid_cell'; + +export interface LogCategoriesGridProps { + dependencies: LogCategoriesGridDependencies; + logCategories: LogCategory[]; +} + +export type LogCategoriesGridDependencies = LogCategoriesGridCellDependencies; + +export const LogCategoriesGrid: React.FC = ({ + dependencies, + logCategories, +}) => { + const [gridState, dispatchGridEvent] = useMachine(gridStateService, { + input: { + visibleColumns: logCategoriesGridColumns.map(({ id }) => id), + }, + inspect: consoleInspector, + }); + + const sortedLogCategories = useMemo(() => { + const sortingCriteria = gridState.context.sortingColumns.map( + ({ id, direction }): [(logCategory: LogCategory) => any, 'asc' | 'desc'] => { + switch (id) { + case 'count': + return [(logCategory: LogCategory) => logCategory.documentCount, direction]; + case 'change_type': + // TODO: use better sorting weight for change types + return [(logCategory: LogCategory) => logCategory.change.type, direction]; + case 'change_time': + return [ + (logCategory: LogCategory) => + 'timestamp' in logCategory.change ? logCategory.change.timestamp ?? '' : '', + direction, + ]; + default: + return [_.identity, direction]; + } + } + ); + return _.orderBy( + logCategories, + sortingCriteria.map(([accessor]) => accessor), + sortingCriteria.map(([, direction]) => direction) + ); + }, [gridState.context.sortingColumns, logCategories]); + + return ( + + dispatchGridEvent({ type: 'changeVisibleColumns', visibleColumns }), + }} + cellContext={createCellContext(sortedLogCategories, dependencies)} + pagination={{ + ...gridState.context.pagination, + onChangeItemsPerPage: (pageSize) => dispatchGridEvent({ type: 'changePageSize', pageSize }), + onChangePage: (pageIndex) => dispatchGridEvent({ type: 'changePageIndex', pageIndex }), + }} + renderCellValue={renderLogCategoriesGridCell} + rowCount={sortedLogCategories.length} + sorting={{ + columns: gridState.context.sortingColumns, + onSort: (sortingColumns) => + dispatchGridEvent({ type: 'changeSortingColumns', sortingColumns }), + }} + /> + ); +}; + +const gridStateService = setup({ + types: { + context: {} as { + visibleColumns: string[]; + pagination: Pick; + sortingColumns: LogCategoriesGridSortingConfig[]; + }, + events: {} as + | { + type: 'changePageSize'; + pageSize: number; + } + | { + type: 'changePageIndex'; + pageIndex: number; + } + | { + type: 'changeSortingColumns'; + sortingColumns: EuiDataGridColumnSortingConfig[]; + } + | { + type: 'changeVisibleColumns'; + visibleColumns: string[]; + }, + input: {} as { + visibleColumns: string[]; + }, + }, +}).createMachine({ + id: 'logCategoriesGridState', + context: ({ input }) => ({ + visibleColumns: input.visibleColumns, + pagination: { pageIndex: 0, pageSize: 20, pageSizeOptions: [10, 20, 50] }, + sortingColumns: [{ id: 'change_time', direction: 'desc' }], + }), + on: { + changePageSize: { + actions: assign(({ context, event }) => ({ + pagination: { + ...context.pagination, + pageIndex: 0, + pageSize: event.pageSize, + }, + })), + }, + changePageIndex: { + actions: assign(({ context, event }) => ({ + pagination: { + ...context.pagination, + pageIndex: event.pageIndex, + }, + })), + }, + changeSortingColumns: { + actions: assign(({ event }) => ({ + sortingColumns: event.sortingColumns.filter( + (sortingConfig): sortingConfig is LogCategoriesGridSortingConfig => + (logCategoriesGridColumnIds as string[]).includes(sortingConfig.id) + ), + })), + }, + changeVisibleColumns: { + actions: assign(({ event }) => ({ + visibleColumns: event.visibleColumns, + })), + }, + }, +}); + +const consoleInspector = createConsoleInspector(); + +const logCategoriesGridLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.euiDataGrid.logCategoriesLabel', + { defaultMessage: 'Log categories' } +); + +interface TypedEuiDataGridColumnSortingConfig + extends EuiDataGridColumnSortingConfig { + id: ColumnId; +} + +type LogCategoriesGridSortingConfig = + TypedEuiDataGridColumnSortingConfig; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_cell.tsx new file mode 100644 index 0000000000000..d6ab4969eaf7b --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_cell.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiDataGridColumn, RenderCellValue } from '@elastic/eui'; +import React from 'react'; +import { LogCategory } from '../../types'; +import { + LogCategoriesGridChangeTimeCell, + LogCategoriesGridChangeTimeCellDependencies, + logCategoriesGridChangeTimeColumn, +} from './log_categories_grid_change_time_cell'; +import { + LogCategoriesGridChangeTypeCell, + logCategoriesGridChangeTypeColumn, +} from './log_categories_grid_change_type_cell'; +import { + LogCategoriesGridCountCell, + logCategoriesGridCountColumn, +} from './log_categories_grid_count_cell'; +import { + LogCategoriesGridHistogramCell, + LogCategoriesGridHistogramCellDependencies, + logCategoriesGridHistoryColumn, +} from './log_categories_grid_histogram_cell'; +import { + LogCategoriesGridPatternCell, + logCategoriesGridPatternColumn, +} from './log_categories_grid_pattern_cell'; + +export interface LogCategoriesGridCellContext { + dependencies: LogCategoriesGridCellDependencies; + logCategories: LogCategory[]; +} + +export type LogCategoriesGridCellDependencies = LogCategoriesGridHistogramCellDependencies & + LogCategoriesGridChangeTimeCellDependencies; + +export const renderLogCategoriesGridCell: RenderCellValue = ({ + rowIndex, + columnId, + isExpanded, + ...rest +}) => { + const { dependencies, logCategories } = getCellContext(rest); + + const logCategory = logCategories[rowIndex]; + + switch (columnId as LogCategoriesGridColumnId) { + case 'pattern': + return ; + case 'count': + return ; + case 'history': + return ( + + ); + case 'change_type': + return ; + case 'change_time': + return ( + + ); + default: + return <>-; + } +}; + +export const logCategoriesGridColumns = [ + logCategoriesGridPatternColumn, + logCategoriesGridCountColumn, + logCategoriesGridChangeTypeColumn, + logCategoriesGridChangeTimeColumn, + logCategoriesGridHistoryColumn, +] satisfies EuiDataGridColumn[]; + +export const logCategoriesGridColumnIds = logCategoriesGridColumns.map(({ id }) => id); + +export type LogCategoriesGridColumnId = (typeof logCategoriesGridColumns)[number]['id']; + +const cellContextKey = 'cellContext'; + +const getCellContext = (cellContext: object): LogCategoriesGridCellContext => + (cellContextKey in cellContext + ? cellContext[cellContextKey] + : {}) as LogCategoriesGridCellContext; + +export const createCellContext = ( + logCategories: LogCategory[], + dependencies: LogCategoriesGridCellDependencies +): { [cellContextKey]: LogCategoriesGridCellContext } => ({ + [cellContextKey]: { + dependencies, + logCategories, + }, +}); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_time_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_time_cell.tsx new file mode 100644 index 0000000000000..5ad8cbdd49346 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_time_cell.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiDataGridColumn } from '@elastic/eui'; +import { SettingsStart } from '@kbn/core-ui-settings-browser'; +import { i18n } from '@kbn/i18n'; +import moment from 'moment'; +import React, { useMemo } from 'react'; +import { LogCategory } from '../../types'; + +export const logCategoriesGridChangeTimeColumn = { + id: 'change_time' as const, + display: i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.changeTimeColumnLabel', + { + defaultMessage: 'Change at', + } + ), + isSortable: true, + initialWidth: 220, + schema: 'datetime', +} satisfies EuiDataGridColumn; + +export interface LogCategoriesGridChangeTimeCellProps { + dependencies: LogCategoriesGridChangeTimeCellDependencies; + logCategory: LogCategory; +} + +export interface LogCategoriesGridChangeTimeCellDependencies { + uiSettings: SettingsStart; +} + +export const LogCategoriesGridChangeTimeCell: React.FC = ({ + dependencies, + logCategory, +}) => { + const dateFormat = useMemo( + () => dependencies.uiSettings.client.get('dateFormat'), + [dependencies.uiSettings.client] + ); + if (!('timestamp' in logCategory.change && logCategory.change.timestamp != null)) { + return null; + } + + if (dateFormat) { + return <>{moment(logCategory.change.timestamp).format(dateFormat)}; + } else { + return <>{logCategory.change.timestamp}; + } +}; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_type_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_type_cell.tsx new file mode 100644 index 0000000000000..af6349bd0e18c --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_change_type_cell.tsx @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiBadge, EuiDataGridColumn } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { LogCategory } from '../../types'; + +export const logCategoriesGridChangeTypeColumn = { + id: 'change_type' as const, + display: i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.changeTypeColumnLabel', + { + defaultMessage: 'Change type', + } + ), + isSortable: true, + initialWidth: 110, +} satisfies EuiDataGridColumn; + +export interface LogCategoriesGridChangeTypeCellProps { + logCategory: LogCategory; +} + +export const LogCategoriesGridChangeTypeCell: React.FC = ({ + logCategory, +}) => { + switch (logCategory.change.type) { + case 'dip': + return {dipBadgeLabel}; + case 'spike': + return {spikeBadgeLabel}; + case 'step': + return {stepBadgeLabel}; + case 'distribution': + return {distributionBadgeLabel}; + case 'rare': + return {rareBadgeLabel}; + case 'trend': + return {trendBadgeLabel}; + case 'other': + return {otherBadgeLabel}; + case 'none': + return <>-; + default: + return {unknownBadgeLabel}; + } +}; + +const dipBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.dipChangeTypeBadgeLabel', + { + defaultMessage: 'Dip', + } +); + +const spikeBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.spikeChangeTypeBadgeLabel', + { + defaultMessage: 'Spike', + } +); + +const stepBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.spikeChangeTypeBadgeLabel', + { + defaultMessage: 'Step', + } +); + +const distributionBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.distributionChangeTypeBadgeLabel', + { + defaultMessage: 'Distribution', + } +); + +const trendBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.spikeChangeTypeBadgeLabel', + { + defaultMessage: 'Trend', + } +); + +const otherBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.otherChangeTypeBadgeLabel', + { + defaultMessage: 'Other', + } +); + +const unknownBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.unknownChangeTypeBadgeLabel', + { + defaultMessage: 'Unknown', + } +); + +const rareBadgeLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.rareChangeTypeBadgeLabel', + { + defaultMessage: 'Rare', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_count_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_count_cell.tsx new file mode 100644 index 0000000000000..f2247aab5212e --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_count_cell.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiDataGridColumn } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedNumber } from '@kbn/i18n-react'; +import React from 'react'; +import { LogCategory } from '../../types'; + +export const logCategoriesGridCountColumn = { + id: 'count' as const, + display: i18n.translate('xpack.observabilityLogsOverview.logCategoriesGrid.countColumnLabel', { + defaultMessage: 'Events', + }), + isSortable: true, + schema: 'numeric', + initialWidth: 100, +} satisfies EuiDataGridColumn; + +export interface LogCategoriesGridCountCellProps { + logCategory: LogCategory; +} + +export const LogCategoriesGridCountCell: React.FC = ({ + logCategory, +}) => { + return ; +}; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_histogram_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_histogram_cell.tsx new file mode 100644 index 0000000000000..2fb50b0f2f3b4 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_histogram_cell.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + BarSeries, + Chart, + LineAnnotation, + LineAnnotationStyle, + PartialTheme, + Settings, + Tooltip, + TooltipType, +} from '@elastic/charts'; +import { EuiDataGridColumn } from '@elastic/eui'; +import { ChartsPluginStart } from '@kbn/charts-plugin/public'; +import { i18n } from '@kbn/i18n'; +import { RecursivePartial } from '@kbn/utility-types'; +import React from 'react'; +import { LogCategory, LogCategoryHistogramBucket } from '../../types'; + +export const logCategoriesGridHistoryColumn = { + id: 'history' as const, + display: i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.histogramColumnLabel', + { + defaultMessage: 'Timeline', + } + ), + isSortable: false, + initialWidth: 250, + isExpandable: false, +} satisfies EuiDataGridColumn; + +export interface LogCategoriesGridHistogramCellProps { + dependencies: LogCategoriesGridHistogramCellDependencies; + logCategory: LogCategory; +} + +export interface LogCategoriesGridHistogramCellDependencies { + charts: ChartsPluginStart; +} + +export const LogCategoriesGridHistogramCell: React.FC = ({ + dependencies: { charts }, + logCategory, +}) => { + const baseTheme = charts.theme.useChartsBaseTheme(); + const sparklineTheme = charts.theme.useSparklineOverrides(); + + return ( + + + + + {'timestamp' in logCategory.change && logCategory.change.timestamp != null ? ( + + ) : null} + + ); +}; + +const localThemeOverrides: PartialTheme = { + scales: { + histogramPadding: 0.1, + }, + background: { + color: 'transparent', + }, +}; + +const annotationStyle: RecursivePartial = { + line: { + strokeWidth: 2, + }, +}; + +const timestampAccessor = (histogram: LogCategoryHistogramBucket) => + new Date(histogram.timestamp).getTime(); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_pattern_cell.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_pattern_cell.tsx new file mode 100644 index 0000000000000..d507487a99e3c --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_grid_pattern_cell.tsx @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiDataGridColumn, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/react'; +import { i18n } from '@kbn/i18n'; +import React, { useMemo } from 'react'; +import { LogCategory } from '../../types'; + +export const logCategoriesGridPatternColumn = { + id: 'pattern' as const, + display: i18n.translate('xpack.observabilityLogsOverview.logCategoriesGrid.patternColumnLabel', { + defaultMessage: 'Pattern', + }), + isSortable: false, + schema: 'string', +} satisfies EuiDataGridColumn; + +export interface LogCategoriesGridPatternCellProps { + logCategory: LogCategory; +} + +export const LogCategoriesGridPatternCell: React.FC = ({ + logCategory, +}) => { + const theme = useEuiTheme(); + const { euiTheme } = theme; + const termsList = useMemo(() => logCategory.terms.split(' '), [logCategory.terms]); + + const commonStyle = css` + display: inline-block; + font-family: ${euiTheme.font.familyCode}; + margin-right: ${euiTheme.size.xs}; + `; + + const termStyle = css` + ${commonStyle}; + `; + + const separatorStyle = css` + ${commonStyle}; + color: ${euiTheme.colors.successText}; + `; + + return ( + 
+      
*

+      {termsList.map((term, index) => (
+        
+          
{term}

+          
*

+        
+      ))}
+
+ ); +}; diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_loading_content.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_loading_content.tsx new file mode 100644 index 0000000000000..0fde469fe717d --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_loading_content.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +export interface LogCategoriesLoadingContentProps { + onCancel?: () => void; + stage: 'counting' | 'categorizing'; +} + +export const LogCategoriesLoadingContent: React.FC = ({ + onCancel, + stage, +}) => { + return ( + } + title={ +

+ {stage === 'counting' + ? logCategoriesLoadingStateCountingTitle + : logCategoriesLoadingStateCategorizingTitle} +

+ } + actions={ + onCancel != null + ? [ + { + onCancel(); + }} + > + {logCategoriesLoadingStateCancelButtonLabel} + , + ] + : [] + } + /> + ); +}; + +const logCategoriesLoadingStateCountingTitle = i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.loadingStageCountingTitle', + { + defaultMessage: 'Estimating log volume', + } +); + +const logCategoriesLoadingStateCategorizingTitle = i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.loadingStageCategorizingTitle', + { + defaultMessage: 'Categorizing logs', + } +); + +const logCategoriesLoadingStateCancelButtonLabel = i18n.translate( + 'xpack.observabilityLogsOverview.logCategoriesGrid.loadingStateCancelButtonLabel', + { + defaultMessage: 'Cancel', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_result_content.tsx b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_result_content.tsx new file mode 100644 index 0000000000000..e16bdda7cb44a --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/log_categories/log_categories_result_content.tsx @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { LogCategory } from '../../types'; +import { IndexNameLogsSourceConfiguration } from '../../utils/logs_source'; +import { + LogCategoriesControlBar, + LogCategoriesControlBarDependencies, +} from './log_categories_control_bar'; +import { LogCategoriesGrid, LogCategoriesGridDependencies } from './log_categories_grid'; + +export interface LogCategoriesResultContentProps { + dependencies: LogCategoriesResultContentDependencies; + documentFilters?: QueryDslQueryContainer[]; + logCategories: LogCategory[]; + logsSource: IndexNameLogsSourceConfiguration; + timeRange: { + start: string; + end: string; + }; +} + +export type LogCategoriesResultContentDependencies = LogCategoriesControlBarDependencies & + LogCategoriesGridDependencies; + +export const LogCategoriesResultContent: React.FC = ({ + dependencies, + documentFilters, + logCategories, + logsSource, + timeRange, +}) => { + if (logCategories.length === 0) { + return ; + } else { + return ( + + + + + + + + + ); + } +}; + +export const LogCategoriesEmptyResultContent: React.FC = () => { + return ( + {emptyResultContentDescription}

} + color="subdued" + layout="horizontal" + title={

{emptyResultContentTitle}

} + titleSize="m" + /> + ); +}; + +const emptyResultContentTitle = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.emptyResultContentTitle', + { + defaultMessage: 'No log categories found', + } +); + +const emptyResultContentDescription = i18n.translate( + 'xpack.observabilityLogsOverview.logCategories.emptyResultContentDescription', + { + defaultMessage: + 'No suitable documents within the time range. Try searching for a longer time period.', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/logs_overview/index.ts b/x-pack/packages/observability/logs_overview/src/components/logs_overview/index.ts new file mode 100644 index 0000000000000..878f634f078ad --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/logs_overview/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './logs_overview'; +export * from './logs_overview_error_content'; +export * from './logs_overview_loading_content'; diff --git a/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview.tsx b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview.tsx new file mode 100644 index 0000000000000..988656eb1571e --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { type LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; +import React from 'react'; +import useAsync from 'react-use/lib/useAsync'; +import { LogsSourceConfiguration, normalizeLogsSource } from '../../utils/logs_source'; +import { LogCategories, LogCategoriesDependencies } from '../log_categories'; +import { LogsOverviewErrorContent } from './logs_overview_error_content'; +import { LogsOverviewLoadingContent } from './logs_overview_loading_content'; + +export interface LogsOverviewProps { + dependencies: LogsOverviewDependencies; + documentFilters?: QueryDslQueryContainer[]; + logsSource?: LogsSourceConfiguration; + timeRange: { + start: string; + end: string; + }; +} + +export type LogsOverviewDependencies = LogCategoriesDependencies & { + logsDataAccess: LogsDataAccessPluginStart; +}; + +export const LogsOverview: React.FC = React.memo( + ({ + dependencies, + documentFilters = defaultDocumentFilters, + logsSource = defaultLogsSource, + timeRange, + }) => { + const normalizedLogsSource = useAsync( + () => normalizeLogsSource({ logsDataAccess: dependencies.logsDataAccess })(logsSource), + [dependencies.logsDataAccess, logsSource] + ); + + if (normalizedLogsSource.loading) { + return ; + } + + if (normalizedLogsSource.error != null || normalizedLogsSource.value == null) { + return ; + } + + return ( + + ); + } +); + +const defaultDocumentFilters: QueryDslQueryContainer[] = []; + +const defaultLogsSource: LogsSourceConfiguration = { type: 'shared_setting' }; diff --git a/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_error_content.tsx b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_error_content.tsx new file mode 100644 index 0000000000000..73586756bb908 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_error_content.tsx @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiCodeBlock, EuiEmptyPrompt } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export interface LogsOverviewErrorContentProps { + error?: Error; +} + +export const LogsOverviewErrorContent: React.FC = ({ error }) => { + return ( + {logsOverviewErrorTitle}} + body={ + +

{error?.stack ?? error?.toString() ?? unknownErrorDescription}

+ + } + layout="vertical" + /> + ); +}; + +const logsOverviewErrorTitle = i18n.translate('xpack.observabilityLogsOverview.errorTitle', { + defaultMessage: 'Error', +}); + +const unknownErrorDescription = i18n.translate( + 'xpack.observabilityLogsOverview.unknownErrorDescription', + { + defaultMessage: 'An unspecified error occurred.', + } +); diff --git a/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_loading_content.tsx b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_loading_content.tsx new file mode 100644 index 0000000000000..7645fdb90f0ac --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/components/logs_overview/logs_overview_loading_content.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export const LogsOverviewLoadingContent: React.FC = ({}) => { + return ( + } + title={

{logsOverviewLoadingTitle}

} + /> + ); +}; + +const logsOverviewLoadingTitle = i18n.translate('xpack.observabilityLogsOverview.loadingTitle', { + defaultMessage: 'Loading', +}); diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_documents.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_documents.ts new file mode 100644 index 0000000000000..7260efe63d435 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_documents.ts @@ -0,0 +1,282 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISearchGeneric } from '@kbn/search-types'; +import { lastValueFrom } from 'rxjs'; +import { fromPromise } from 'xstate5'; +import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; +import { z } from '@kbn/zod'; +import { LogCategorizationParams } from './types'; +import { createCategorizationRequestParams } from './queries'; +import { LogCategory, LogCategoryChange } from '../../types'; + +// the fraction of a category's histogram below which the category is considered rare +const rarityThreshold = 0.2; +const maxCategoriesCount = 1000; + +export const categorizeDocuments = ({ search }: { search: ISearchGeneric }) => + fromPromise< + { + categories: LogCategory[]; + hasReachedLimit: boolean; + }, + LogCategorizationParams & { + samplingProbability: number; + ignoredCategoryTerms: string[]; + minDocsPerCategory: number; + } + >( + async ({ + input: { + index, + endTimestamp, + startTimestamp, + timeField, + messageField, + samplingProbability, + ignoredCategoryTerms, + documentFilters = [], + minDocsPerCategory, + }, + signal, + }) => { + const randomSampler = createRandomSamplerWrapper({ + probability: samplingProbability, + seed: 1, + }); + + const requestParams = createCategorizationRequestParams({ + index, + timeField, + messageField, + startTimestamp, + endTimestamp, + randomSampler, + additionalFilters: documentFilters, + ignoredCategoryTerms, + minDocsPerCategory, + maxCategoriesCount, + }); + + const { rawResponse } = await lastValueFrom( + search({ params: requestParams }, { abortSignal: signal }) + ); + + if (rawResponse.aggregations == null) { + throw new Error('No aggregations found in large categories response'); + } + + const logCategoriesAggResult = randomSampler.unwrap(rawResponse.aggregations); + + if (!('categories' in logCategoriesAggResult)) { + throw new Error('No categorization aggregation found in large categories response'); + } + + const logCategories = + (logCategoriesAggResult.categories.buckets as unknown[]).map(mapCategoryBucket) ?? []; + + return { + categories: logCategories, + hasReachedLimit: logCategories.length >= maxCategoriesCount, + }; + } + ); + +const mapCategoryBucket = (bucket: any): LogCategory => + esCategoryBucketSchema + .transform((parsedBucket) => ({ + change: mapChangePoint(parsedBucket), + documentCount: parsedBucket.doc_count, + histogram: parsedBucket.histogram, + terms: parsedBucket.key, + })) + .parse(bucket); + +const mapChangePoint = ({ change, histogram }: EsCategoryBucket): LogCategoryChange => { + switch (change.type) { + case 'stationary': + if (isRareInHistogram(histogram)) { + return { + type: 'rare', + timestamp: findFirstNonZeroBucket(histogram)?.timestamp ?? histogram[0].timestamp, + }; + } else { + return { + type: 'none', + }; + } + case 'dip': + case 'spike': + return { + type: change.type, + timestamp: change.bucket.key, + }; + case 'step_change': + return { + type: 'step', + timestamp: change.bucket.key, + }; + case 'distribution_change': + return { + type: 'distribution', + timestamp: change.bucket.key, + }; + case 'trend_change': + return { + type: 'trend', + timestamp: change.bucket.key, + correlationCoefficient: change.details.r_value, + }; + case 'unknown': + return { + type: 'unknown', + rawChange: change.rawChange, + }; + case 'non_stationary': + default: + return { + type: 'other', + }; + } +}; + +/** + * The official types are lacking the change_point aggregation + */ +const esChangePointBucketSchema = z.object({ + key: z.string().datetime(), + doc_count: z.number(), +}); + +const esChangePointDetailsSchema = z.object({ + p_value: z.number(), +}); + +const esChangePointCorrelationSchema = esChangePointDetailsSchema.extend({ + r_value: z.number(), +}); + +const esChangePointSchema = z.union([ + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + dip: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { dip: details } }) => ({ + type: 'dip' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + spike: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { spike: details } }) => ({ + type: 'spike' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + step_change: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { step_change: details } }) => ({ + type: 'step_change' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + trend_change: esChangePointCorrelationSchema, + }), + }) + .transform(({ bucket, type: { trend_change: details } }) => ({ + type: 'trend_change' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + distribution_change: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { distribution_change: details } }) => ({ + type: 'distribution_change' as const, + bucket, + details, + })), + z + .object({ + type: z.object({ + non_stationary: esChangePointCorrelationSchema.extend({ + trend: z.enum(['increasing', 'decreasing']), + }), + }), + }) + .transform(({ type: { non_stationary: details } }) => ({ + type: 'non_stationary' as const, + details, + })), + z + .object({ + type: z.object({ + stationary: z.object({}), + }), + }) + .transform(() => ({ type: 'stationary' as const })), + z + .object({ + type: z.object({}), + }) + .transform((value) => ({ type: 'unknown' as const, rawChange: JSON.stringify(value) })), +]); + +const esHistogramSchema = z + .object({ + buckets: z.array( + z + .object({ + key_as_string: z.string(), + doc_count: z.number(), + }) + .transform((bucket) => ({ + timestamp: bucket.key_as_string, + documentCount: bucket.doc_count, + })) + ), + }) + .transform(({ buckets }) => buckets); + +type EsHistogram = z.output; + +const esCategoryBucketSchema = z.object({ + key: z.string(), + doc_count: z.number(), + change: esChangePointSchema, + histogram: esHistogramSchema, +}); + +type EsCategoryBucket = z.output; + +const isRareInHistogram = (histogram: EsHistogram): boolean => + histogram.filter((bucket) => bucket.documentCount > 0).length < + histogram.length * rarityThreshold; + +const findFirstNonZeroBucket = (histogram: EsHistogram) => + histogram.find((bucket) => bucket.documentCount > 0); diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_logs_service.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_logs_service.ts new file mode 100644 index 0000000000000..deeb758d2d737 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/categorize_logs_service.ts @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MachineImplementationsFrom, assign, setup } from 'xstate5'; +import { LogCategory } from '../../types'; +import { getPlaceholderFor } from '../../utils/xstate5_utils'; +import { categorizeDocuments } from './categorize_documents'; +import { countDocuments } from './count_documents'; +import { CategorizeLogsServiceDependencies, LogCategorizationParams } from './types'; + +export const categorizeLogsService = setup({ + types: { + input: {} as LogCategorizationParams, + output: {} as { + categories: LogCategory[]; + documentCount: number; + hasReachedLimit: boolean; + samplingProbability: number; + }, + context: {} as { + categories: LogCategory[]; + documentCount: number; + error?: Error; + hasReachedLimit: boolean; + parameters: LogCategorizationParams; + samplingProbability: number; + }, + events: {} as { + type: 'cancel'; + }, + }, + actors: { + countDocuments: getPlaceholderFor(countDocuments), + categorizeDocuments: getPlaceholderFor(categorizeDocuments), + }, + actions: { + storeError: assign((_, params: { error: unknown }) => ({ + error: params.error instanceof Error ? params.error : new Error(String(params.error)), + })), + storeCategories: assign( + ({ context }, params: { categories: LogCategory[]; hasReachedLimit: boolean }) => ({ + categories: [...context.categories, ...params.categories], + hasReachedLimit: params.hasReachedLimit, + }) + ), + storeDocumentCount: assign( + (_, params: { documentCount: number; samplingProbability: number }) => ({ + documentCount: params.documentCount, + samplingProbability: params.samplingProbability, + }) + ), + }, + guards: { + hasTooFewDocuments: (_guardArgs, params: { documentCount: number }) => params.documentCount < 1, + requiresSampling: (_guardArgs, params: { samplingProbability: number }) => + params.samplingProbability < 1, + }, +}).createMachine({ + /** @xstate-layout N4IgpgJg5mDOIC5QGMCGAXMUD2AnAlgF5gAy2UsAdMtgK4B26+9UAItsrQLZiOwDEEbPTCVmAN2wBrUWkw4CxMhWp1GzNh2690sBBI4Z8wgNoAGALrmLiUAAdssfE2G2QAD0QBmMwA5KACy+AQFmob4AjABMwQBsADQgAJ6IkYEAnJkA7FmxZlERmQGxAL4liXJYeESk5FQ0DEws7Jw8fILCogYy1BhVirUqDerNWm26+vSScsb01iYRNkggDk4u9G6eCD7+QSFhftFxiSkIvgCsWZSxEVlRsbFZ52Zm515lFX0KNcr1ak2aVo6ARCERiKbSWRfapKOqqRoaFraPiTaZGUyWExRJb2RzOWabbx+QLBULhI7FE7eWL+F45GnRPIRZkfECVb6wob-RFjYH8MC4XB4Sh2AA2GAAZnguL15DDBn8EaMgSiDDMMVZLG5VvjXMstjsSftyTFKclEOdzgFKF5zukvA8zBFnl50udWez5b94SNAcjdPw0PRkGBRdZtXj1oTtsS9mTDqaEuaEBF8udKFkIr5fK6olkzOksgEPdCBt6JWB0MgABYaADKqC4YsgAGFS-g4B0wd0oXKBg2m6LW+24OHljqo-rEMzbpQos8-K7fC9CknTrF0rEbbb0oVMoWIgF3eU2e3OVQK1XaywB82IG2+x2BAKhbgReL0FLcDLPf3G3eH36J8x1xNYCSnFNmSuecXhzdJlydTcqQQLJfHSOc0PyLJN3SMxYiPEtH3PShLxret-yHe8RwEIMQzDLVx0jcDQC2GdoIXOCENXZDsyiOcAiiKJ0iiPDLi8V1CKA4jSOvKAACUwC4VBmA0QDvk7UEughHpfxqBSlJUlg1OqUcGNA3UNggrMs347IjzdaIvGQwSvECXI8k3Z43gEiJJI5BUSMrMiWH05T6FU6j+UFYUxUlaVZSksBQsMqBjIIUycRWJi9RY6dIn8KIAjsu1zkc5CAmiG1fBiaIzB8B0QmPT4iICmSNGS8KjMi2jQxArKwJyjw8pswriocqInOTLwIi3ASD1yQpswCd5WXobAIDgNxdPPCMBss3KEAAWjXRBDvTfcLsu9Jlr8r04WGAEkXGeBGL26MBOQzIt2ut4cwmirCt8W6yzhNqbwo4dH0216LOjTMIjnBdYhK1DYgdHjihtZbUIdWIXJuYGflBoLZI6iKoZe8zJwOw9KtGt1kbuTcsmQrwi0oeCQjzZ5blwt1Cek5TKN22GIIKZbAgKC45pyLyeLwtz4Kyabs1QgWAs0kXqaGhBxdcnzpaE2XXmch0MORmaBJeLwjbKMogA */ + id: 'categorizeLogs', + context: ({ input }) => ({ + categories: [], + documentCount: 0, + hasReachedLimit: false, + parameters: input, + samplingProbability: 1, + }), + initial: 'countingDocuments', + states: { + countingDocuments: { + invoke: { + src: 'countDocuments', + input: ({ context }) => context.parameters, + onDone: [ + { + target: 'done', + guard: { + type: 'hasTooFewDocuments', + params: ({ event }) => event.output, + }, + actions: [ + { + type: 'storeDocumentCount', + params: ({ event }) => event.output, + }, + ], + }, + { + target: 'fetchingSampledCategories', + guard: { + type: 'requiresSampling', + params: ({ event }) => event.output, + }, + actions: [ + { + type: 'storeDocumentCount', + params: ({ event }) => event.output, + }, + ], + }, + { + target: 'fetchingRemainingCategories', + actions: [ + { + type: 'storeDocumentCount', + params: ({ event }) => event.output, + }, + ], + }, + ], + onError: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: ({ event }) => ({ error: event.error }), + }, + ], + }, + }, + + on: { + cancel: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: () => ({ error: new Error('Counting cancelled') }), + }, + ], + }, + }, + }, + + fetchingSampledCategories: { + invoke: { + src: 'categorizeDocuments', + id: 'categorizeSampledCategories', + input: ({ context }) => ({ + ...context.parameters, + samplingProbability: context.samplingProbability, + ignoredCategoryTerms: [], + minDocsPerCategory: 10, + }), + onDone: { + target: 'fetchingRemainingCategories', + actions: [ + { + type: 'storeCategories', + params: ({ event }) => event.output, + }, + ], + }, + onError: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: ({ event }) => ({ error: event.error }), + }, + ], + }, + }, + + on: { + cancel: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: () => ({ error: new Error('Categorization cancelled') }), + }, + ], + }, + }, + }, + + fetchingRemainingCategories: { + invoke: { + src: 'categorizeDocuments', + id: 'categorizeRemainingCategories', + input: ({ context }) => ({ + ...context.parameters, + samplingProbability: 1, + ignoredCategoryTerms: context.categories.map((category) => category.terms), + minDocsPerCategory: 0, + }), + onDone: { + target: 'done', + actions: [ + { + type: 'storeCategories', + params: ({ event }) => event.output, + }, + ], + }, + onError: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: ({ event }) => ({ error: event.error }), + }, + ], + }, + }, + + on: { + cancel: { + target: 'failed', + actions: [ + { + type: 'storeError', + params: () => ({ error: new Error('Categorization cancelled') }), + }, + ], + }, + }, + }, + + failed: { + type: 'final', + }, + + done: { + type: 'final', + }, + }, + output: ({ context }) => ({ + categories: context.categories, + documentCount: context.documentCount, + hasReachedLimit: context.hasReachedLimit, + samplingProbability: context.samplingProbability, + }), +}); + +export const createCategorizeLogsServiceImplementations = ({ + search, +}: CategorizeLogsServiceDependencies): MachineImplementationsFrom< + typeof categorizeLogsService +> => ({ + actors: { + categorizeDocuments: categorizeDocuments({ search }), + countDocuments: countDocuments({ search }), + }, +}); diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/count_documents.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/count_documents.ts new file mode 100644 index 0000000000000..359f9ddac2bd8 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/count_documents.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getSampleProbability } from '@kbn/ml-random-sampler-utils'; +import { ISearchGeneric } from '@kbn/search-types'; +import { lastValueFrom } from 'rxjs'; +import { fromPromise } from 'xstate5'; +import { LogCategorizationParams } from './types'; +import { createCategorizationQuery } from './queries'; + +export const countDocuments = ({ search }: { search: ISearchGeneric }) => + fromPromise< + { + documentCount: number; + samplingProbability: number; + }, + LogCategorizationParams + >( + async ({ + input: { index, endTimestamp, startTimestamp, timeField, messageField, documentFilters }, + signal, + }) => { + const { rawResponse: totalHitsResponse } = await lastValueFrom( + search( + { + params: { + index, + size: 0, + track_total_hits: true, + query: createCategorizationQuery({ + messageField, + timeField, + startTimestamp, + endTimestamp, + additionalFilters: documentFilters, + }), + }, + }, + { abortSignal: signal } + ) + ); + + const documentCount = + totalHitsResponse.hits.total == null + ? 0 + : typeof totalHitsResponse.hits.total === 'number' + ? totalHitsResponse.hits.total + : totalHitsResponse.hits.total.value; + const samplingProbability = getSampleProbability(documentCount); + + return { + documentCount, + samplingProbability, + }; + } + ); diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/index.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/index.ts new file mode 100644 index 0000000000000..149359b7d2015 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './categorize_logs_service'; diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/queries.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/queries.ts new file mode 100644 index 0000000000000..aef12da303bcc --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/queries.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { calculateAuto } from '@kbn/calculate-auto'; +import { RandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; +import moment from 'moment'; + +const isoTimestampFormat = "YYYY-MM-DD'T'HH:mm:ss.SSS'Z'"; + +export const createCategorizationQuery = ({ + messageField, + timeField, + startTimestamp, + endTimestamp, + additionalFilters = [], + ignoredCategoryTerms = [], +}: { + messageField: string; + timeField: string; + startTimestamp: string; + endTimestamp: string; + additionalFilters?: QueryDslQueryContainer[]; + ignoredCategoryTerms?: string[]; +}): QueryDslQueryContainer => { + return { + bool: { + filter: [ + { + exists: { + field: messageField, + }, + }, + { + range: { + [timeField]: { + gte: startTimestamp, + lte: endTimestamp, + format: 'strict_date_time', + }, + }, + }, + ...additionalFilters, + ], + must_not: ignoredCategoryTerms.map(createCategoryQuery(messageField)), + }, + }; +}; + +export const createCategorizationRequestParams = ({ + index, + timeField, + messageField, + startTimestamp, + endTimestamp, + randomSampler, + minDocsPerCategory = 0, + additionalFilters = [], + ignoredCategoryTerms = [], + maxCategoriesCount = 1000, +}: { + startTimestamp: string; + endTimestamp: string; + index: string; + timeField: string; + messageField: string; + randomSampler: RandomSamplerWrapper; + minDocsPerCategory?: number; + additionalFilters?: QueryDslQueryContainer[]; + ignoredCategoryTerms?: string[]; + maxCategoriesCount?: number; +}) => { + const startMoment = moment(startTimestamp, isoTimestampFormat); + const endMoment = moment(endTimestamp, isoTimestampFormat); + const fixedIntervalDuration = calculateAuto.atLeast( + 24, + moment.duration(endMoment.diff(startMoment)) + ); + const fixedIntervalSize = `${Math.ceil(fixedIntervalDuration?.asMinutes() ?? 1)}m`; + + return { + index, + size: 0, + track_total_hits: false, + query: createCategorizationQuery({ + messageField, + timeField, + startTimestamp, + endTimestamp, + additionalFilters, + ignoredCategoryTerms, + }), + aggs: randomSampler.wrap({ + histogram: { + date_histogram: { + field: timeField, + fixed_interval: fixedIntervalSize, + extended_bounds: { + min: startTimestamp, + max: endTimestamp, + }, + }, + }, + categories: { + categorize_text: { + field: messageField, + size: maxCategoriesCount, + categorization_analyzer: { + tokenizer: 'standard', + }, + ...(minDocsPerCategory > 0 ? { min_doc_count: minDocsPerCategory } : {}), + }, + aggs: { + histogram: { + date_histogram: { + field: timeField, + fixed_interval: fixedIntervalSize, + extended_bounds: { + min: startTimestamp, + max: endTimestamp, + }, + }, + }, + change: { + // @ts-expect-error the official types don't support the change_point aggregation + change_point: { + buckets_path: 'histogram>_count', + }, + }, + }, + }, + }), + }; +}; + +export const createCategoryQuery = + (messageField: string) => + (categoryTerms: string): QueryDslQueryContainer => ({ + match: { + [messageField]: { + query: categoryTerms, + operator: 'AND' as const, + fuzziness: 0, + auto_generate_synonyms_phrase_query: false, + }, + }, + }); diff --git a/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/types.ts b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/types.ts new file mode 100644 index 0000000000000..e094317a98d62 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/services/categorize_logs_service/types.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { ISearchGeneric } from '@kbn/search-types'; + +export interface CategorizeLogsServiceDependencies { + search: ISearchGeneric; +} + +export interface LogCategorizationParams { + documentFilters: QueryDslQueryContainer[]; + endTimestamp: string; + index: string; + messageField: string; + startTimestamp: string; + timeField: string; +} diff --git a/x-pack/packages/observability/logs_overview/src/types.ts b/x-pack/packages/observability/logs_overview/src/types.ts new file mode 100644 index 0000000000000..4c3d27eca7e7c --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/types.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface LogCategory { + change: LogCategoryChange; + documentCount: number; + histogram: LogCategoryHistogramBucket[]; + terms: string; +} + +export type LogCategoryChange = + | LogCategoryNoChange + | LogCategoryRareChange + | LogCategorySpikeChange + | LogCategoryDipChange + | LogCategoryStepChange + | LogCategoryDistributionChange + | LogCategoryTrendChange + | LogCategoryOtherChange + | LogCategoryUnknownChange; + +export interface LogCategoryNoChange { + type: 'none'; +} + +export interface LogCategoryRareChange { + type: 'rare'; + timestamp: string; +} + +export interface LogCategorySpikeChange { + type: 'spike'; + timestamp: string; +} + +export interface LogCategoryDipChange { + type: 'dip'; + timestamp: string; +} + +export interface LogCategoryStepChange { + type: 'step'; + timestamp: string; +} + +export interface LogCategoryTrendChange { + type: 'trend'; + timestamp: string; + correlationCoefficient: number; +} + +export interface LogCategoryDistributionChange { + type: 'distribution'; + timestamp: string; +} + +export interface LogCategoryOtherChange { + type: 'other'; + timestamp?: string; +} + +export interface LogCategoryUnknownChange { + type: 'unknown'; + rawChange: string; +} + +export interface LogCategoryHistogramBucket { + documentCount: number; + timestamp: string; +} diff --git a/x-pack/packages/observability/logs_overview/src/utils/logs_source.ts b/x-pack/packages/observability/logs_overview/src/utils/logs_source.ts new file mode 100644 index 0000000000000..0c8767c8702d4 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/utils/logs_source.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type AbstractDataView } from '@kbn/data-views-plugin/common'; +import { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; + +export type LogsSourceConfiguration = + | SharedSettingLogsSourceConfiguration + | IndexNameLogsSourceConfiguration + | DataViewLogsSourceConfiguration; + +export interface SharedSettingLogsSourceConfiguration { + type: 'shared_setting'; + timestampField?: string; + messageField?: string; +} + +export interface IndexNameLogsSourceConfiguration { + type: 'index_name'; + indexName: string; + timestampField: string; + messageField: string; +} + +export interface DataViewLogsSourceConfiguration { + type: 'data_view'; + dataView: AbstractDataView; + messageField?: string; +} + +export const normalizeLogsSource = + ({ logsDataAccess }: { logsDataAccess: LogsDataAccessPluginStart }) => + async (logsSource: LogsSourceConfiguration): Promise => { + switch (logsSource.type) { + case 'index_name': + return logsSource; + case 'shared_setting': + const logSourcesFromSharedSettings = + await logsDataAccess.services.logSourcesService.getLogSources(); + return { + type: 'index_name', + indexName: logSourcesFromSharedSettings + .map((logSource) => logSource.indexPattern) + .join(','), + timestampField: logsSource.timestampField ?? '@timestamp', + messageField: logsSource.messageField ?? 'message', + }; + case 'data_view': + return { + type: 'index_name', + indexName: logsSource.dataView.getIndexPattern(), + timestampField: logsSource.dataView.timeFieldName ?? '@timestamp', + messageField: logsSource.messageField ?? 'message', + }; + } + }; diff --git a/x-pack/packages/observability/logs_overview/src/utils/xstate5_utils.ts b/x-pack/packages/observability/logs_overview/src/utils/xstate5_utils.ts new file mode 100644 index 0000000000000..3df0bf4ea3988 --- /dev/null +++ b/x-pack/packages/observability/logs_overview/src/utils/xstate5_utils.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getPlaceholderFor = any>( + implementationFactory: ImplementationFactory +): ReturnType => + (() => { + throw new Error('Not implemented'); + }) as ReturnType; diff --git a/x-pack/packages/observability/logs_overview/tsconfig.json b/x-pack/packages/observability/logs_overview/tsconfig.json new file mode 100644 index 0000000000000..886062ae8855f --- /dev/null +++ b/x-pack/packages/observability/logs_overview/tsconfig.json @@ -0,0 +1,39 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react", + "@kbn/ambient-ui-types", + "@kbn/ambient-storybook-types", + "@emotion/react/types/css-prop" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/data-views-plugin", + "@kbn/i18n", + "@kbn/search-types", + "@kbn/xstate-utils", + "@kbn/core-ui-settings-browser", + "@kbn/i18n-react", + "@kbn/charts-plugin", + "@kbn/utility-types", + "@kbn/logs-data-access-plugin", + "@kbn/ml-random-sampler-utils", + "@kbn/zod", + "@kbn/calculate-auto", + "@kbn/discover-plugin", + "@kbn/es-query", + "@kbn/router-utils", + "@kbn/share-plugin", + ] +} diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/service_logs/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/service_logs/index.tsx index 4df52758ceda3..a1dadbf186b91 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/service_logs/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/service_logs/index.tsx @@ -5,19 +5,36 @@ * 2.0. */ -import React from 'react'; +import React, { useMemo } from 'react'; import moment from 'moment'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { LogStream } from '@kbn/logs-shared-plugin/public'; import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values'; -import { useFetcher } from '../../../hooks/use_fetcher'; -import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; -import { APIReturnType } from '../../../services/rest/create_call_apm_api'; - import { CONTAINER_ID, SERVICE_ENVIRONMENT, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; +import { useKibana } from '../../../context/kibana_context/use_kibana'; import { useAnyOfApmParams } from '../../../hooks/use_apm_params'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { useTimeRange } from '../../../hooks/use_time_range'; +import { APIReturnType } from '../../../services/rest/create_call_apm_api'; export function ServiceLogs() { + const { + services: { + logsShared: { LogsOverview }, + }, + } = useKibana(); + + const isLogsOverviewEnabled = LogsOverview.useIsEnabled(); + + if (isLogsOverviewEnabled) { + return ; + } else { + return ; + } +} + +export function ClassicServiceLogsStream() { const { serviceName } = useApmServiceContext(); const { @@ -58,6 +75,54 @@ export function ServiceLogs() { ); } +export function ServiceLogsOverview() { + const { + services: { logsShared }, + } = useKibana(); + const { serviceName } = useApmServiceContext(); + const { + query: { environment, kuery, rangeFrom, rangeTo }, + } = useAnyOfApmParams('/services/{serviceName}/logs'); + const { start, end } = useTimeRange({ rangeFrom, rangeTo }); + const timeRange = useMemo(() => ({ start, end }), [start, end]); + + const { data: logFilters, status } = useFetcher( + async (callApmApi) => { + if (start == null || end == null) { + return; + } + + const { containerIds } = await callApmApi( + 'GET /internal/apm/services/{serviceName}/infrastructure_attributes', + { + params: { + path: { serviceName }, + query: { + environment, + kuery, + start, + end, + }, + }, + } + ); + + return [getInfrastructureFilter({ containerIds, environment, serviceName })]; + }, + [environment, kuery, serviceName, start, end] + ); + + if (status === FETCH_STATUS.SUCCESS) { + return ; + } else if (status === FETCH_STATUS.FAILURE) { + return ( + + ); + } else { + return ; + } +} + export function getInfrastructureKQLFilter({ data, serviceName, @@ -84,3 +149,99 @@ export function getInfrastructureKQLFilter({ return [serviceNameAndEnvironmentCorrelation, ...containerIdCorrelation].join(' or '); } + +export function getInfrastructureFilter({ + containerIds, + environment, + serviceName, +}: { + containerIds: string[]; + environment: string; + serviceName: string; +}): QueryDslQueryContainer { + return { + bool: { + should: [ + ...getServiceShouldClauses({ environment, serviceName }), + ...getContainerShouldClauses({ containerIds }), + ], + minimum_should_match: 1, + }, + }; +} + +export function getServiceShouldClauses({ + environment, + serviceName, +}: { + environment: string; + serviceName: string; +}): QueryDslQueryContainer[] { + const serviceNameFilter: QueryDslQueryContainer = { + term: { + [SERVICE_NAME]: serviceName, + }, + }; + + if (environment === ENVIRONMENT_ALL.value) { + return [serviceNameFilter]; + } else { + return [ + { + bool: { + filter: [ + serviceNameFilter, + { + term: { + [SERVICE_ENVIRONMENT]: environment, + }, + }, + ], + }, + }, + { + bool: { + filter: [serviceNameFilter], + must_not: [ + { + exists: { + field: SERVICE_ENVIRONMENT, + }, + }, + ], + }, + }, + ]; + } +} + +export function getContainerShouldClauses({ + containerIds = [], +}: { + containerIds: string[]; +}): QueryDslQueryContainer[] { + if (containerIds.length === 0) { + return []; + } + + return [ + { + bool: { + filter: [ + { + terms: { + [CONTAINER_ID]: containerIds, + }, + }, + ], + must_not: [ + { + term: { + [SERVICE_NAME]: '*', + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/apm/public/components/routing/service_detail/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/routing/service_detail/index.tsx index d746e0464fd40..8a4a1c32877c5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/routing/service_detail/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/routing/service_detail/index.tsx @@ -330,7 +330,7 @@ export const serviceDetailRoute = { }), element: , searchBarOptions: { - showUnifiedSearchBar: false, + showQueryInput: false, }, }), '/services/{serviceName}/infrastructure': { diff --git a/x-pack/plugins/observability_solution/apm/public/plugin.ts b/x-pack/plugins/observability_solution/apm/public/plugin.ts index 9a9f45f42a39e..b21bdedac9ef8 100644 --- a/x-pack/plugins/observability_solution/apm/public/plugin.ts +++ b/x-pack/plugins/observability_solution/apm/public/plugin.ts @@ -69,6 +69,7 @@ import { from } from 'rxjs'; import { map } from 'rxjs'; import type { CloudSetup } from '@kbn/cloud-plugin/public'; import type { ServerlessPluginStart } from '@kbn/serverless/public'; +import { LogsSharedClientStartExports } from '@kbn/logs-shared-plugin/public'; import type { ConfigSchema } from '.'; import { registerApmRuleTypes } from './components/alerting/rule_types/register_apm_rule_types'; import { registerEmbeddables } from './embeddable/register_embeddables'; @@ -142,6 +143,7 @@ export interface ApmPluginStartDeps { dashboard: DashboardStart; metricsDataAccess: MetricsDataPluginStart; uiSettings: IUiSettingsClient; + logsShared: LogsSharedClientStartExports; } const applicationsTitle = i18n.translate('xpack.apm.navigation.rootTitle', { diff --git a/x-pack/plugins/observability_solution/infra/public/pages/metrics/hosts/components/tabs/logs/logs_tab_content.tsx b/x-pack/plugins/observability_solution/infra/public/pages/metrics/hosts/components/tabs/logs/logs_tab_content.tsx index 27344ccd1f108..68a5db6d4d484 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/metrics/hosts/components/tabs/logs/logs_tab_content.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/metrics/hosts/components/tabs/logs/logs_tab_content.tsx @@ -5,21 +5,37 @@ * 2.0. */ -import React, { useMemo } from 'react'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { LogStream } from '@kbn/logs-shared-plugin/public'; -import { i18n } from '@kbn/i18n'; +import React, { useMemo } from 'react'; import { InfraLoadingPanel } from '../../../../../../components/loading'; +import { useKibanaContextForPlugin } from '../../../../../../hooks/use_kibana'; +import { useLogViewReference } from '../../../../../../hooks/use_log_view_reference'; +import { buildCombinedAssetFilter } from '../../../../../../utils/filters/build'; import { useHostsViewContext } from '../../../hooks/use_hosts_view'; -import { useUnifiedSearchContext } from '../../../hooks/use_unified_search'; import { useLogsSearchUrlState } from '../../../hooks/use_logs_search_url_state'; +import { useUnifiedSearchContext } from '../../../hooks/use_unified_search'; import { LogsLinkToStream } from './logs_link_to_stream'; import { LogsSearchBar } from './logs_search_bar'; -import { buildCombinedAssetFilter } from '../../../../../../utils/filters/build'; -import { useLogViewReference } from '../../../../../../hooks/use_log_view_reference'; export const LogsTabContent = () => { + const { + services: { + logsShared: { LogsOverview }, + }, + } = useKibanaContextForPlugin(); + const isLogsOverviewEnabled = LogsOverview.useIsEnabled(); + if (isLogsOverviewEnabled) { + return ; + } else { + return ; + } +}; + +export const LogsTabLogStreamContent = () => { const [filterQuery] = useLogsSearchUrlState(); const { getDateRangeAsTimestamp } = useUnifiedSearchContext(); const { from, to } = useMemo(() => getDateRangeAsTimestamp(), [getDateRangeAsTimestamp]); @@ -53,22 +69,7 @@ export const LogsTabContent = () => { }, [filterQuery.query, hostNodes]); if (loading || logViewLoading || !logView) { - return ( - - - - } - /> - - - ); + return ; } return ( @@ -112,3 +113,53 @@ const createHostsFilterQueryParam = (hostNodes: string[]): string => { return hostsQueryParam; }; + +const LogsTabLogsOverviewContent = () => { + const { + services: { + logsShared: { LogsOverview }, + }, + } = useKibanaContextForPlugin(); + + const { parsedDateRange } = useUnifiedSearchContext(); + const timeRange = useMemo( + () => ({ start: parsedDateRange.from, end: parsedDateRange.to }), + [parsedDateRange.from, parsedDateRange.to] + ); + + const { hostNodes, loading, error } = useHostsViewContext(); + const logFilters = useMemo( + () => [ + buildCombinedAssetFilter({ + field: 'host.name', + values: hostNodes.map((p) => p.name), + }).query as QueryDslQueryContainer, + ], + [hostNodes] + ); + + if (loading) { + return ; + } else if (error != null) { + return ; + } else { + return ; + } +}; + +const LogsTabLoadingContent = () => ( + + + + } + /> + + +); diff --git a/x-pack/plugins/observability_solution/logs_shared/kibana.jsonc b/x-pack/plugins/observability_solution/logs_shared/kibana.jsonc index ea93fd326dac7..10c8fe32cfe9c 100644 --- a/x-pack/plugins/observability_solution/logs_shared/kibana.jsonc +++ b/x-pack/plugins/observability_solution/logs_shared/kibana.jsonc @@ -9,13 +9,14 @@ "browser": true, "configPath": ["xpack", "logs_shared"], "requiredPlugins": [ + "charts", "data", "dataViews", "discoverShared", - "usageCollection", + "logsDataAccess", "observabilityShared", "share", - "logsDataAccess" + "usageCollection", ], "optionalPlugins": [ "observabilityAIAssistant", diff --git a/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/index.tsx b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/index.tsx new file mode 100644 index 0000000000000..627cdc8447eea --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './logs_overview'; diff --git a/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.mock.tsx b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.mock.tsx new file mode 100644 index 0000000000000..435766bff793d --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.mock.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { + LogsOverviewProps, + SelfContainedLogsOverviewComponent, + SelfContainedLogsOverviewHelpers, +} from './logs_overview'; + +export const createLogsOverviewMock = () => { + const LogsOverviewMock = jest.fn(LogsOverviewMockImpl) as unknown as ILogsOverviewMock; + + LogsOverviewMock.useIsEnabled = jest.fn(() => true); + + LogsOverviewMock.ErrorContent = jest.fn(() =>
); + + LogsOverviewMock.LoadingContent = jest.fn(() =>
); + + return LogsOverviewMock; +}; + +const LogsOverviewMockImpl = (_props: LogsOverviewProps) => { + return
; +}; + +type ILogsOverviewMock = jest.Mocked & + jest.Mocked; diff --git a/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.tsx b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.tsx new file mode 100644 index 0000000000000..7b60aee5be57c --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_shared/public/components/logs_overview/logs_overview.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID } from '@kbn/management-settings-ids'; +import type { + LogsOverviewProps as FullLogsOverviewProps, + LogsOverviewDependencies, + LogsOverviewErrorContentProps, +} from '@kbn/observability-logs-overview'; +import { dynamic } from '@kbn/shared-ux-utility'; +import React from 'react'; +import useObservable from 'react-use/lib/useObservable'; + +const LazyLogsOverview = dynamic(() => + import('@kbn/observability-logs-overview').then((mod) => ({ default: mod.LogsOverview })) +); + +const LazyLogsOverviewErrorContent = dynamic(() => + import('@kbn/observability-logs-overview').then((mod) => ({ + default: mod.LogsOverviewErrorContent, + })) +); + +const LazyLogsOverviewLoadingContent = dynamic(() => + import('@kbn/observability-logs-overview').then((mod) => ({ + default: mod.LogsOverviewLoadingContent, + })) +); + +export type LogsOverviewProps = Omit; + +export interface SelfContainedLogsOverviewHelpers { + useIsEnabled: () => boolean; + ErrorContent: React.ComponentType; + LoadingContent: React.ComponentType; +} + +export type SelfContainedLogsOverviewComponent = React.ComponentType; + +export type SelfContainedLogsOverview = SelfContainedLogsOverviewComponent & + SelfContainedLogsOverviewHelpers; + +export const createLogsOverview = ( + dependencies: LogsOverviewDependencies +): SelfContainedLogsOverview => { + const SelfContainedLogsOverview = (props: LogsOverviewProps) => { + return ; + }; + + const isEnabled$ = dependencies.uiSettings.client.get$( + OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID, + defaultIsEnabled + ); + + SelfContainedLogsOverview.useIsEnabled = (): boolean => { + return useObservable(isEnabled$, defaultIsEnabled); + }; + + SelfContainedLogsOverview.ErrorContent = LazyLogsOverviewErrorContent; + + SelfContainedLogsOverview.LoadingContent = LazyLogsOverviewLoadingContent; + + return SelfContainedLogsOverview; +}; + +const defaultIsEnabled = false; diff --git a/x-pack/plugins/observability_solution/logs_shared/public/index.ts b/x-pack/plugins/observability_solution/logs_shared/public/index.ts index a602b25786116..3d601c9936f2d 100644 --- a/x-pack/plugins/observability_solution/logs_shared/public/index.ts +++ b/x-pack/plugins/observability_solution/logs_shared/public/index.ts @@ -50,6 +50,7 @@ export type { UpdatedDateRange, VisibleInterval, } from './components/logging/log_text_stream/scrollable_log_text_stream_view'; +export type { LogsOverviewProps } from './components/logs_overview'; export const WithSummary = dynamic(() => import('./containers/logs/log_summary/with_summary')); export const LogEntryFlyout = dynamic( diff --git a/x-pack/plugins/observability_solution/logs_shared/public/mocks.tsx b/x-pack/plugins/observability_solution/logs_shared/public/mocks.tsx index a9b0ebd6a6aa3..ffb867abbcc17 100644 --- a/x-pack/plugins/observability_solution/logs_shared/public/mocks.tsx +++ b/x-pack/plugins/observability_solution/logs_shared/public/mocks.tsx @@ -6,12 +6,14 @@ */ import { createLogAIAssistantMock } from './components/log_ai_assistant/log_ai_assistant.mock'; +import { createLogsOverviewMock } from './components/logs_overview/logs_overview.mock'; import { createLogViewsServiceStartMock } from './services/log_views/log_views_service.mock'; import { LogsSharedClientStartExports } from './types'; export const createLogsSharedPluginStartMock = (): jest.Mocked => ({ logViews: createLogViewsServiceStartMock(), LogAIAssistant: createLogAIAssistantMock(), + LogsOverview: createLogsOverviewMock(), }); export const _ensureTypeCompatibility = (): LogsSharedClientStartExports => diff --git a/x-pack/plugins/observability_solution/logs_shared/public/plugin.ts b/x-pack/plugins/observability_solution/logs_shared/public/plugin.ts index d6f4ac81fe266..fc17e9b17cc82 100644 --- a/x-pack/plugins/observability_solution/logs_shared/public/plugin.ts +++ b/x-pack/plugins/observability_solution/logs_shared/public/plugin.ts @@ -12,6 +12,7 @@ import { TraceLogsLocatorDefinition, } from '../common/locators'; import { createLogAIAssistant, createLogsAIAssistantRenderer } from './components/log_ai_assistant'; +import { createLogsOverview } from './components/logs_overview'; import { LogViewsService } from './services/log_views'; import { LogsSharedClientCoreSetup, @@ -51,8 +52,16 @@ export class LogsSharedPlugin implements LogsSharedClientPluginClass { } public start(core: CoreStart, plugins: LogsSharedClientStartDeps) { - const { http } = core; - const { data, dataViews, discoverShared, observabilityAIAssistant, logsDataAccess } = plugins; + const { http, settings } = core; + const { + charts, + data, + dataViews, + discoverShared, + logsDataAccess, + observabilityAIAssistant, + share, + } = plugins; const logViews = this.logViews.start({ http, @@ -61,9 +70,18 @@ export class LogsSharedPlugin implements LogsSharedClientPluginClass { search: data.search, }); + const LogsOverview = createLogsOverview({ + charts, + logsDataAccess, + search: data.search.search, + uiSettings: settings, + share, + }); + if (!observabilityAIAssistant) { return { logViews, + LogsOverview, }; } @@ -77,6 +95,7 @@ export class LogsSharedPlugin implements LogsSharedClientPluginClass { return { logViews, LogAIAssistant, + LogsOverview, }; } diff --git a/x-pack/plugins/observability_solution/logs_shared/public/types.ts b/x-pack/plugins/observability_solution/logs_shared/public/types.ts index 58b180ee8b6ef..4237c28c621b8 100644 --- a/x-pack/plugins/observability_solution/logs_shared/public/types.ts +++ b/x-pack/plugins/observability_solution/logs_shared/public/types.ts @@ -5,19 +5,19 @@ * 2.0. */ +import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; import type { CoreSetup, CoreStart, Plugin as PluginClass } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { DiscoverSharedPublicStart } from '@kbn/discover-shared-plugin/public'; -import { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; +import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public'; import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public'; import type { UiActionsStart } from '@kbn/ui-actions-plugin/public'; - -import { LogsSharedLocators } from '../common/locators'; +import type { LogsSharedLocators } from '../common/locators'; import type { LogAIAssistantProps } from './components/log_ai_assistant/log_ai_assistant'; -// import type { OsqueryPluginStart } from '../../osquery/public'; -import { LogViewsServiceSetup, LogViewsServiceStart } from './services/log_views'; +import type { SelfContainedLogsOverview } from './components/logs_overview'; +import type { LogViewsServiceSetup, LogViewsServiceStart } from './services/log_views'; // Our own setup and start contract values export interface LogsSharedClientSetupExports { @@ -28,6 +28,7 @@ export interface LogsSharedClientSetupExports { export interface LogsSharedClientStartExports { logViews: LogViewsServiceStart; LogAIAssistant?: (props: Omit) => JSX.Element; + LogsOverview: SelfContainedLogsOverview; } export interface LogsSharedClientSetupDeps { @@ -35,6 +36,7 @@ export interface LogsSharedClientSetupDeps { } export interface LogsSharedClientStartDeps { + charts: ChartsPluginStart; data: DataPublicPluginStart; dataViews: DataViewsPublicPluginStart; discoverShared: DiscoverSharedPublicStart; diff --git a/x-pack/plugins/observability_solution/logs_shared/server/feature_flags.ts b/x-pack/plugins/observability_solution/logs_shared/server/feature_flags.ts new file mode 100644 index 0000000000000..0298416bd3f26 --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_shared/server/feature_flags.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { UiSettingsParams } from '@kbn/core-ui-settings-common'; +import { i18n } from '@kbn/i18n'; +import { OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID } from '@kbn/management-settings-ids'; + +const technicalPreviewLabel = i18n.translate('xpack.logsShared.technicalPreviewSettingLabel', { + defaultMessage: 'Technical Preview', +}); + +export const featureFlagUiSettings: Record = { + [OBSERVABILITY_LOGS_SHARED_NEW_LOGS_OVERVIEW_ID]: { + category: ['observability'], + name: i18n.translate('xpack.logsShared.newLogsOverviewSettingName', { + defaultMessage: 'New logs overview', + }), + value: false, + description: i18n.translate('xpack.logsShared.newLogsOverviewSettingDescription', { + defaultMessage: '{technicalPreviewLabel} Enable the new logs overview experience.', + + values: { technicalPreviewLabel: `[${technicalPreviewLabel}]` }, + }), + type: 'boolean', + schema: schema.boolean(), + requiresPageReload: true, + }, +}; diff --git a/x-pack/plugins/observability_solution/logs_shared/server/plugin.ts b/x-pack/plugins/observability_solution/logs_shared/server/plugin.ts index 7c97e175ed64f..d1f6399104fc2 100644 --- a/x-pack/plugins/observability_solution/logs_shared/server/plugin.ts +++ b/x-pack/plugins/observability_solution/logs_shared/server/plugin.ts @@ -5,8 +5,19 @@ * 2.0. */ -import { PluginInitializerContext, CoreStart, Plugin, Logger } from '@kbn/core/server'; - +import { CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import { defaultLogViewId } from '../common/log_views'; +import { LogsSharedConfig } from '../common/plugin_config'; +import { registerDeprecations } from './deprecations'; +import { featureFlagUiSettings } from './feature_flags'; +import { KibanaFramework } from './lib/adapters/framework/kibana_framework_adapter'; +import { LogsSharedKibanaLogEntriesAdapter } from './lib/adapters/log_entries/kibana_log_entries_adapter'; +import { LogsSharedLogEntriesDomain } from './lib/domains/log_entries_domain'; +import { LogsSharedBackendLibs, LogsSharedDomainLibs } from './lib/logs_shared_types'; +import { initLogsSharedServer } from './logs_shared_server'; +import { logViewSavedObjectType } from './saved_objects'; +import { LogEntriesService } from './services/log_entries'; +import { LogViewsService } from './services/log_views'; import { LogsSharedPluginCoreSetup, LogsSharedPluginSetup, @@ -15,17 +26,6 @@ import { LogsSharedServerPluginStartDeps, UsageCollector, } from './types'; -import { logViewSavedObjectType } from './saved_objects'; -import { initLogsSharedServer } from './logs_shared_server'; -import { LogViewsService } from './services/log_views'; -import { KibanaFramework } from './lib/adapters/framework/kibana_framework_adapter'; -import { LogsSharedBackendLibs, LogsSharedDomainLibs } from './lib/logs_shared_types'; -import { LogsSharedLogEntriesDomain } from './lib/domains/log_entries_domain'; -import { LogsSharedKibanaLogEntriesAdapter } from './lib/adapters/log_entries/kibana_log_entries_adapter'; -import { LogEntriesService } from './services/log_entries'; -import { LogsSharedConfig } from '../common/plugin_config'; -import { registerDeprecations } from './deprecations'; -import { defaultLogViewId } from '../common/log_views'; export class LogsSharedPlugin implements @@ -88,6 +88,8 @@ export class LogsSharedPlugin registerDeprecations({ core }); + core.uiSettings.register(featureFlagUiSettings); + return { ...domainLibs, logViews, diff --git a/x-pack/plugins/observability_solution/logs_shared/tsconfig.json b/x-pack/plugins/observability_solution/logs_shared/tsconfig.json index 38cbba7c252c0..788f55c9b6fc5 100644 --- a/x-pack/plugins/observability_solution/logs_shared/tsconfig.json +++ b/x-pack/plugins/observability_solution/logs_shared/tsconfig.json @@ -44,5 +44,9 @@ "@kbn/logs-data-access-plugin", "@kbn/core-deprecations-common", "@kbn/core-deprecations-server", + "@kbn/management-settings-ids", + "@kbn/observability-logs-overview", + "@kbn/charts-plugin", + "@kbn/core-ui-settings-common", ] } diff --git a/yarn.lock b/yarn.lock index 54a38b2c0e5d3..019de6121540e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5879,6 +5879,10 @@ version "0.0.0" uid "" +"@kbn/observability-logs-overview@link:x-pack/packages/observability/logs_overview": + version "0.0.0" + uid "" + "@kbn/observability-onboarding-e2e@link:x-pack/plugins/observability_solution/observability_onboarding/e2e": version "0.0.0" uid "" @@ -12105,6 +12109,14 @@ use-isomorphic-layout-effect "^1.1.2" use-sync-external-store "^1.0.0" +"@xstate5/react@npm:@xstate/react@^4.1.2": + version "4.1.2" + resolved "https://registry.yarnpkg.com/@xstate/react/-/react-4.1.2.tgz#4bfcdf2d9e9ef1eaea7388d1896649345e6679cd" + integrity sha512-orAidFrKCrU0ZwN5l/ABPlBfW2ziRDT2RrYoktRlZ0WRoLvA2E/uAC1JpZt43mCLtc8jrdwYCgJiqx1V8NvGTw== + dependencies: + use-isomorphic-layout-effect "^1.1.2" + use-sync-external-store "^1.2.0" + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -32800,6 +32812,11 @@ xpath@^0.0.33: resolved "https://registry.yarnpkg.com/xpath/-/xpath-0.0.33.tgz#5136b6094227c5df92002e7c3a13516a5074eb07" integrity sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA== +"xstate5@npm:xstate@^5.18.1", xstate@^5.18.1: + version "5.18.1" + resolved "https://registry.yarnpkg.com/xstate/-/xstate-5.18.1.tgz#c4d43ceaba6e6c31705d36bd96e285de4be4f7f4" + integrity sha512-m02IqcCQbaE/kBQLunwub/5i8epvkD2mFutnL17Oeg1eXTShe1sRF4D5mhv1dlaFO4vbW5gRGRhraeAD5c938g== + xstate@^4.38.2: version "4.38.2" resolved "https://registry.yarnpkg.com/xstate/-/xstate-4.38.2.tgz#1b74544fc9c8c6c713ba77f81c6017e65aa89804"