From 34f21becf521f4a369c566ec9893afc16b8d5030 Mon Sep 17 00:00:00 2001 From: Kerry Gallagher Date: Tue, 23 May 2023 15:52:55 +0100 Subject: [PATCH] [Logs+] Implement Data Source APIs (#156413) ## Summary Research: https://github.com/elastic/observability-dev/issues/2620 Requirements: https://github.com/elastic/observability-dev/issues/2639 Implementation: https://github.com/elastic/observability-dev/issues/2653 This implements two APIs ("Enumerate integrations" and "Enumerate data streams") under the Fleet plugin, and specifically the `/epm` namespace given the input here: https://github.com/elastic/observability-dev/issues/2639#issuecomment-1521689096 The Enumerate Integrations API can be queried like so (example with all parameters): `GET /api/fleet/epm/packages/installed?nameQuery=system&pageSize=5&type=logs&pageAfter=["system"]&sortDirection=asc` The Enumerate Data Streams API can be queried like so (example with all parameters): `GET /api/fleet/epm/data_streams?uncategorisedOnly=true&type=logs&sortDirection=desc&datasetQuery=beat` --- .../plugins/fleet/common/constants/routes.ts | 3 + .../fleet/common/types/models/data_stream.ts | 2 + .../fleet/common/types/rest_spec/epm.ts | 23 +++ .../fleet/server/routes/epm/handlers.ts | 52 +++++ .../plugins/fleet/server/routes/epm/index.ts | 26 +++ .../services/epm/data_streams/get.test.ts | 191 ++++++++++++++++++ .../server/services/epm/data_streams/get.ts | 47 +++++ .../server/services/epm/data_streams/index.ts | 8 + .../server/services/epm/packages/get.test.ts | 166 ++++++++++++++- .../fleet/server/services/epm/packages/get.ts | 128 +++++++++++- .../server/services/epm/packages/index.ts | 1 + .../fleet/server/types/rest_spec/epm.ts | 39 ++++ .../apis/epm/data_stream.ts | 10 + .../fleet_api_integration/apis/epm/get.ts | 56 +++++ .../fleet_api_integration/apis/epm/setup.ts | 12 +- 15 files changed, 761 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/data_streams/get.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/data_streams/get.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/data_streams/index.ts diff --git a/x-pack/plugins/fleet/common/constants/routes.ts b/x-pack/plugins/fleet/common/constants/routes.ts index 7b77296c4e52b..9c1415df8a650 100644 --- a/x-pack/plugins/fleet/common/constants/routes.ts +++ b/x-pack/plugins/fleet/common/constants/routes.ts @@ -21,14 +21,17 @@ export const LIMITED_CONCURRENCY_ROUTE_TAG = 'ingest:limited-concurrency'; // EPM API routes const EPM_PACKAGES_MANY = `${EPM_API_ROOT}/packages`; +const EPM_PACKAGES_INSTALLED = `${EPM_API_ROOT}/packages/installed`; const EPM_PACKAGES_BULK = `${EPM_PACKAGES_MANY}/_bulk`; const EPM_PACKAGES_ONE_DEPRECATED = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { BULK_INSTALL_PATTERN: EPM_PACKAGES_BULK, LIST_PATTERN: EPM_PACKAGES_MANY, + INSTALLED_LIST_PATTERN: EPM_PACKAGES_INSTALLED, LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`, INFO_PATTERN: EPM_PACKAGES_ONE, + DATA_STREAMS_PATTERN: `${EPM_API_ROOT}/data_streams`, INSTALL_FROM_REGISTRY_PATTERN: EPM_PACKAGES_ONE, INSTALL_BY_UPLOAD_PATTERN: EPM_PACKAGES_MANY, DELETE_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/fleet/common/types/models/data_stream.ts b/x-pack/plugins/fleet/common/types/models/data_stream.ts index 353502cc658cc..ab01f82626f0d 100644 --- a/x-pack/plugins/fleet/common/types/models/data_stream.ts +++ b/x-pack/plugins/fleet/common/types/models/data_stream.ts @@ -24,3 +24,5 @@ export interface DataStream { serviceName: string; } | null; } + +export type PackageDataStreamTypes = 'logs' | 'metrics' | 'traces' | 'synthetics' | 'profiling'; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index d7321d8218590..e9df09206ca6f 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { SortResults } from '@elastic/elasticsearch/lib/api/types'; + import type { AssetReference, CategorySummaryList, @@ -13,6 +15,7 @@ import type { PackageUsageStats, InstallType, InstallSource, + EpmPackageInstallStatus, } from '../models/epm'; export interface GetCategoriesRequest { @@ -46,6 +49,26 @@ export interface GetPackagesResponse { response?: PackageList; } +interface InstalledPackage { + name: string; + version: string; + status: EpmPackageInstallStatus; + dataStreams: Array<{ + name: string; + title: string; + }>; +} +export interface GetInstalledPackagesResponse { + items: InstalledPackage[]; + total: number; + searchAfter?: SortResults; +} + +export interface GetEpmDataStreamsResponse { + items: Array<{ + name: string; + }>; +} export interface GetLimitedPackagesResponse { items: string[]; // deprecated in 8.0 diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 0f2232593fd5e..1e528b8cc2a10 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -30,10 +30,14 @@ import type { GetStatsResponse, UpdatePackageResponse, GetVerificationKeyIdResponse, + GetInstalledPackagesResponse, + GetEpmDataStreamsResponse, } from '../../../common/types'; import type { GetCategoriesRequestSchema, GetPackagesRequestSchema, + GetInstalledPackagesRequestSchema, + GetDataStreamsRequestSchema, GetFileRequestSchema, GetInfoRequestSchema, InstallPackageFromRegistryRequestSchema, @@ -49,6 +53,7 @@ import { bulkInstallPackages, getCategories, getPackages, + getInstalledPackages, getFile, getPackageInfo, isBulkInstallError, @@ -66,6 +71,7 @@ import { getPackageUsageStats } from '../../services/epm/packages/get'; import { updatePackage } from '../../services/epm/packages/update'; import { getGpgKeyIdOrUndefined } from '../../services/epm/packages/package_verification'; import type { ReauthorizeTransformRequestSchema } from '../../types'; +import { getDataStreams } from '../../services/epm/data_streams'; const CACHE_CONTROL_10_MINUTES_HEADER: HttpResponseOptions['headers'] = { 'cache-control': 'max-age=600', @@ -115,6 +121,52 @@ export const getListHandler: FleetRequestHandler< } }; +export const getInstalledListHandler: FleetRequestHandler< + undefined, + TypeOf +> = async (context, request, response) => { + try { + const savedObjectsClient = (await context.fleet).internalSoClient; + const res = await getInstalledPackages({ + savedObjectsClient, + ...request.query, + }); + + const body: GetInstalledPackagesResponse = { ...res }; + + return response.ok({ + body, + }); + } catch (error) { + return defaultFleetErrorHandler({ error, response }); + } +}; + +export const getDataStreamsHandler: FleetRequestHandler< + undefined, + TypeOf +> = async (context, request, response) => { + try { + const coreContext = await context.core; + // Query datastreams as the current user as the Kibana internal user may not have all the required permissions + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const res = await getDataStreams({ + esClient, + ...request.query, + }); + + const body: GetEpmDataStreamsResponse = { + ...res, + }; + + return response.ok({ + body, + }); + } catch (error) { + return defaultFleetErrorHandler({ error, response }); + } +}; + export const getLimitedListHandler: FleetRequestHandler< undefined, TypeOf, diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index b707a8b80e582..133f5b21eeb7c 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -27,6 +27,7 @@ import { splitPkgKey } from '../../services/epm/registry'; import { GetCategoriesRequestSchema, GetPackagesRequestSchema, + GetInstalledPackagesRequestSchema, GetFileRequestSchema, GetInfoRequestSchema, GetInfoRequestSchemaDeprecated, @@ -40,11 +41,13 @@ import { UpdatePackageRequestSchema, UpdatePackageRequestSchemaDeprecated, ReauthorizeTransformRequestSchema, + GetDataStreamsRequestSchema, } from '../../types'; import { getCategoriesHandler, getListHandler, + getInstalledListHandler, getLimitedListHandler, getFileHandler, getInfoHandler, @@ -56,6 +59,7 @@ import { updatePackageHandler, getVerificationKeyIdHandler, reauthorizeTransformsHandler, + getDataStreamsHandler, } from './handlers'; const MAX_FILE_SIZE_BYTES = 104857600; // 100MB @@ -83,6 +87,17 @@ export const registerRoutes = (router: FleetAuthzRouter) => { getListHandler ); + router.get( + { + path: EPM_API_ROUTES.INSTALLED_LIST_PATTERN, + validate: GetInstalledPackagesRequestSchema, + fleetAuthz: { + integrations: { readPackageInfo: true }, + }, + }, + getInstalledListHandler + ); + router.get( { path: EPM_API_ROUTES.LIMITED_LIST_PATTERN, @@ -201,6 +216,17 @@ export const registerRoutes = (router: FleetAuthzRouter) => { getVerificationKeyIdHandler ); + router.get( + { + path: EPM_API_ROUTES.DATA_STREAMS_PATTERN, + validate: GetDataStreamsRequestSchema, + fleetAuthz: { + integrations: { readPackageInfo: true }, + }, + }, + getDataStreamsHandler + ); + // deprecated since 8.0 router.get( { diff --git a/x-pack/plugins/fleet/server/services/epm/data_streams/get.test.ts b/x-pack/plugins/fleet/server/services/epm/data_streams/get.test.ts new file mode 100644 index 0000000000000..0d0d49f6b2917 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/data_streams/get.test.ts @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + +import { dataStreamService } from '../../data_streams'; + +import { getDataStreams } from './get'; + +jest.mock('../../data_streams', () => { + return { + dataStreamService: { + getMatchingDataStreams: jest.fn().mockImplementation(() => { + return [ + { + name: 'logs-elastic_agent-default', + timestamp_field: { name: '@timestamp' }, + indices: [ + { + index_name: '.ds-logs-elastic_agent-default-2023.05.17-000001', + index_uuid: 'EcqQR36PTNCKVnfAftq_Rw', + }, + ], + generation: 1, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'elastic_agent' } }, + status: 'YELLOW', + template: 'logs-elastic_agent', + ilm_policy: 'logs', + hidden: false, + system: false, + allow_custom_routing: false, + replicated: false, + }, + { + name: 'logs-elastic_agent.filebeat-default', + timestamp_field: { name: '@timestamp' }, + indices: [ + { + index_name: '.ds-logs-elastic_agent.filebeat-default-2023.05.17-000001', + index_uuid: 'v5uEn55TRrurU3Bf4CBtzw', + }, + ], + generation: 1, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'elastic_agent' } }, + status: 'YELLOW', + template: 'logs-elastic_agent.filebeat', + ilm_policy: 'logs', + hidden: false, + system: false, + allow_custom_routing: false, + replicated: false, + }, + { + name: 'logs-elastic_agent.fleet_server-default', + timestamp_field: { name: '@timestamp' }, + indices: [ + { + index_name: '.ds-logs-elastic_agent.fleet_server-default-2023.05.17-000001', + index_uuid: 'nThe6dkaQnagAlyNsAyYsA', + }, + ], + generation: 1, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'elastic_agent' } }, + status: 'YELLOW', + template: 'logs-elastic_agent.fleet_server', + ilm_policy: 'logs', + hidden: false, + system: false, + allow_custom_routing: false, + replicated: false, + }, + { + name: 'logs-elastic_agent.metricbeat-default', + timestamp_field: { name: '@timestamp' }, + indices: [ + { + index_name: '.ds-logs-elastic_agent.metricbeat-default-2023.05.17-000001', + index_uuid: 'Y5vQ7V6-QSSMM-CPdqOkCg', + }, + ], + generation: 1, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'elastic_agent' } }, + status: 'YELLOW', + template: 'logs-elastic_agent.metricbeat', + ilm_policy: 'logs', + hidden: false, + system: false, + allow_custom_routing: false, + replicated: false, + }, + { + name: 'logs-test.test-default', + timestamp_field: { name: '@timestamp' }, + indices: [ + { + index_name: '.ds-logs-elastic_agent.metricbeat-default-2023.05.17-000001', + index_uuid: 'Y5vQ7V6-QSSMM-CPdqOkCg', + }, + ], + }, + ]; + }), + }, + }; +}); + +describe('getDataStreams', () => { + it('Passes the correct parameters to the DataStreamService', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'asc', + uncategorisedOnly: true, + }); + expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith(expect.anything(), { + type: 'logs', + dataset: '*nginx*', + }); + }); + describe('uncategorisedOnly option', () => { + it('Returns the correct number of results when true', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const results = await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'asc', + uncategorisedOnly: true, + }); + expect(results.items.length).toBe(1); + }); + it('Returns the correct number of results when false', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const results = await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'asc', + uncategorisedOnly: false, + }); + expect(results.items.length).toBe(5); + }); + }); + describe('Can be sorted', () => { + it('Ascending', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const results = await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'asc', + uncategorisedOnly: false, + }); + expect(results.items[0].name).toBe('logs-elastic_agent-default'); + }); + it('Descending', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const results = await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'desc', + uncategorisedOnly: false, + }); + expect(results.items[0].name).toBe('logs-test.test-default'); + }); + }); + it('Formats the items correctly', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const results = await getDataStreams({ + esClient: esClientMock, + type: 'logs', + datasetQuery: 'nginx', + sortOrder: 'desc', + uncategorisedOnly: false, + }); + expect(results.items).toEqual([ + { name: 'logs-test.test-default' }, + { name: 'logs-elastic_agent.metricbeat-default' }, + { name: 'logs-elastic_agent.fleet_server-default' }, + { name: 'logs-elastic_agent.filebeat-default' }, + { name: 'logs-elastic_agent-default' }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/data_streams/get.ts b/x-pack/plugins/fleet/server/services/epm/data_streams/get.ts new file mode 100644 index 0000000000000..562a8280ca051 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/data_streams/get.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; + +import type { PackageDataStreamTypes } from '../../../../common/types'; + +import { dataStreamService } from '../../data_streams'; + +export async function getDataStreams(options: { + esClient: ElasticsearchClient; + type?: PackageDataStreamTypes; + datasetQuery?: string; + sortOrder: 'asc' | 'desc'; + uncategorisedOnly: boolean; +}) { + const { esClient, type, datasetQuery, uncategorisedOnly, sortOrder } = options; + + const allDataStreams = await dataStreamService.getMatchingDataStreams(esClient, { + type: type ? type : '*', + dataset: datasetQuery ? `*${datasetQuery}*` : '*', + }); + + const filteredDataStreams = uncategorisedOnly + ? allDataStreams.filter((stream) => { + return !stream._meta || !stream._meta.managed_by || stream._meta.managed_by !== 'fleet'; + }) + : allDataStreams; + + const mappedDataStreams = filteredDataStreams.map((dataStream) => { + return { name: dataStream.name }; + }); + + const sortedDataStreams = mappedDataStreams.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + + const dataStreams = sortOrder === 'asc' ? sortedDataStreams : sortedDataStreams.reverse(); + + return { + items: dataStreams, + }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/data_streams/index.ts b/x-pack/plugins/fleet/server/services/epm/data_streams/index.ts new file mode 100644 index 0000000000000..4b54bdcbf33de --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/data_streams/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getDataStreams } from './get'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 14fe84796b9fd..f7e3161f8ad3c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -23,7 +23,7 @@ import { auditLoggingService } from '../../audit_logging'; import * as Registry from '../registry'; -import { getPackageInfo, getPackages, getPackageUsageStats } from './get'; +import { getInstalledPackages, getPackageInfo, getPackages, getPackageUsageStats } from './get'; jest.mock('../registry'); jest.mock('../../settings'); @@ -342,6 +342,170 @@ owner: elastic`, }); }); + describe('getInstalledPackages', () => { + it('Passes the correct parameters to the SavedObjects client', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValue({ + saved_objects: [ + { + type: 'epm-packages', + id: 'elastic_agent', + attributes: { + es_index_patterns: { + apm_server_logs: 'logs-elastic_agent.apm_server-*', + apm_server_metrics: 'metrics-elastic_agent.apm_server-*', + }, + name: 'elastic_agent', + version: '1.7.0', + install_status: 'installed', + }, + references: [], + }, + ], + } as any); + + await getInstalledPackages({ + savedObjectsClient: soClient, + dataStreamType: 'logs', + nameQuery: 'nginx', + searchAfter: ['system'], + perPage: 10, + sortOrder: 'asc', + }); + expect(soClient.find).toHaveBeenCalledWith({ + filter: { + arguments: [ + { + arguments: [ + { + isQuoted: false, + type: 'literal', + value: 'epm-packages.attributes.install_status', + }, + { + isQuoted: false, + type: 'literal', + value: 'installed', + }, + ], + function: 'is', + type: 'function', + }, + { + arguments: [ + { + isQuoted: false, + type: 'literal', + value: 'epm-packages.attributes.installed_es', + }, + { + arguments: [ + { + isQuoted: false, + type: 'literal', + value: 'type', + }, + { + isQuoted: false, + type: 'literal', + value: 'index_template', + }, + ], + function: 'is', + type: 'function', + }, + ], + function: 'nested', + type: 'function', + }, + { + arguments: [ + { + isQuoted: false, + type: 'literal', + value: 'epm-packages.attributes.installed_es', + }, + { + arguments: [ + { + isQuoted: false, + type: 'literal', + value: 'id', + }, + { + type: 'wildcard', + value: 'logs-@kuery-wildcard@', + }, + ], + function: 'is', + type: 'function', + }, + ], + function: 'nested', + type: 'function', + }, + ], + function: 'and', + type: 'function', + }, + perPage: 10, + search: 'nginx* | nginx', + searchAfter: ['system'], + searchFields: ['name'], + sortField: 'name', + sortOrder: 'asc', + type: 'epm-packages', + }); + }); + it('Formats items correctly', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValue({ + total: 5, + saved_objects: [ + { + type: 'epm-packages', + id: 'elastic_agent', + attributes: { + es_index_patterns: { + apm_server_logs: 'logs-elastic_agent.apm_server-*', + apm_server_metrics: 'metrics-elastic_agent.apm_server-*', + }, + name: 'elastic_agent', + version: '1.7.0', + install_status: 'installed', + }, + references: [], + sort: ['elastic_agent'], + }, + ], + } as any); + + const results = await getInstalledPackages({ + savedObjectsClient: soClient, + dataStreamType: 'logs', + nameQuery: 'nginx', + searchAfter: ['system'], + perPage: 10, + sortOrder: 'asc', + }); + + expect(results).toEqual({ + items: [ + { + dataStreams: [{ name: 'logs-elastic_agent.apm_server-*', title: 'apm_server_logs' }], + name: 'elastic_agent', + status: 'installed', + version: '1.7.0', + }, + ], + searchAfter: ['elastic_agent'], + total: 5, + }); + }); + }); + describe('getPackageInfo', () => { beforeEach(() => { const mockContract = createAppContextStartContractMock(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 696aa6a272970..3a7b70a175a19 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -10,13 +10,24 @@ import semverGte from 'semver/functions/gte'; import type { Logger } from '@kbn/core/server'; import { withSpan } from '@kbn/apm-utils'; +import type { SortResults } from '@elastic/elasticsearch/lib/api/types'; + +import { nodeBuilder } from '@kbn/es-query'; + +import { buildNode as buildFunctionNode } from '@kbn/es-query/src/kuery/node_types/function'; +import { buildNode as buildWildcardNode } from '@kbn/es-query/src/kuery/node_types/wildcard'; + import { installationStatuses, PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT, } from '../../../../common/constants'; import { isPackageLimited } from '../../../../common/services'; -import type { PackageUsageStats, Installable } from '../../../../common/types'; +import type { + PackageUsageStats, + Installable, + PackageDataStreamTypes, +} from '../../../../common/types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { ArchivePackage, @@ -138,6 +149,48 @@ export async function getPackages( return packageListWithoutStatus; } +interface GetInstalledPackagesOptions { + savedObjectsClient: SavedObjectsClientContract; + dataStreamType?: PackageDataStreamTypes; + nameQuery?: string; + searchAfter?: SortResults; + perPage: number; + sortOrder: 'asc' | 'desc'; +} +export async function getInstalledPackages(options: GetInstalledPackagesOptions) { + const { savedObjectsClient, ...otherOptions } = options; + const { dataStreamType } = otherOptions; + + const packageSavedObjects = await getInstalledPackageSavedObjects( + savedObjectsClient, + otherOptions + ); + + const integrations = packageSavedObjects.saved_objects.map((integrationSavedObject) => { + const { + name, + version, + install_status: installStatus, + es_index_patterns: esIndexPatterns, + } = integrationSavedObject.attributes; + + const dataStreams = getInstalledPackageSavedObjectDataStreams(esIndexPatterns, dataStreamType); + + return { + name, + version, + status: installStatus, + dataStreams, + }; + }); + + return { + items: integrations, + total: packageSavedObjects.total, + searchAfter: packageSavedObjects.saved_objects.at(-1)?.sort, // Enable ability to use searchAfter in subsequent queries + }; +} + // Get package names for packages which cannot have more than one package policy on an agent policy export async function getLimitedPackages(options: { savedObjectsClient: SavedObjectsClientContract; @@ -195,6 +248,79 @@ export async function getPackageSavedObjects( return result; } +export async function getInstalledPackageSavedObjects( + savedObjectsClient: SavedObjectsClientContract, + options: Omit +) { + const { searchAfter, sortOrder, perPage, nameQuery, dataStreamType } = options; + + const result = await savedObjectsClient.find({ + type: PACKAGES_SAVED_OBJECT_TYPE, + // Pagination + perPage, + ...(searchAfter && { searchAfter }), + // Sort + sortField: 'name', + sortOrder, + // Name filter + ...(nameQuery && { searchFields: ['name'] }), + ...(nameQuery && { search: `${nameQuery}* | ${nameQuery}` }), + filter: nodeBuilder.and([ + // Filter to installed packages only + nodeBuilder.is( + `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_status`, + installationStatuses.Installed + ), + // Filter for a "queryable" marker + buildFunctionNode( + 'nested', + `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.installed_es`, + nodeBuilder.is('type', 'index_template') + ), + // "Type" filter + ...(dataStreamType + ? [ + buildFunctionNode( + 'nested', + `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.installed_es`, + nodeBuilder.is('id', buildWildcardNode(`${dataStreamType}-*`)) + ), + ] + : []), + ]), + }); + + for (const savedObject of result.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: savedObject.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + } + + return result; +} + +function getInstalledPackageSavedObjectDataStreams( + indexPatterns: Record, + dataStreamType?: string +) { + return Object.entries(indexPatterns) + .map(([key, value]) => { + return { + name: value, + title: key, + }; + }) + .filter((stream) => { + if (!dataStreamType) { + return true; + } else { + return stream.name.startsWith(`${dataStreamType}-`); + } + }); +} + export const getInstallations = getPackageSavedObjects; export async function getPackageInfo({ diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index 7538212fb9efe..1141100b10e1a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -22,6 +22,7 @@ export { getInstallations, getPackageInfo, getPackages, + getInstalledPackages, getLimitedPackages, } from './get'; diff --git a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts index 204183a056b1b..a3d371149ac4c 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts @@ -24,6 +24,45 @@ export const GetPackagesRequestSchema = { }), }; +export const GetInstalledPackagesRequestSchema = { + query: schema.object({ + dataStreamType: schema.maybe( + schema.oneOf([ + schema.literal('logs'), + schema.literal('metrics'), + schema.literal('traces'), + schema.literal('synthetics'), + schema.literal('profiling'), + ]) + ), + nameQuery: schema.maybe(schema.string()), + searchAfter: schema.maybe(schema.arrayOf(schema.oneOf([schema.string(), schema.number()]))), + perPage: schema.number({ defaultValue: 30 }), + sortOrder: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { + defaultValue: 'asc', + }), + }), +}; + +export const GetDataStreamsRequestSchema = { + query: schema.object({ + type: schema.maybe( + schema.oneOf([ + schema.literal('logs'), + schema.literal('metrics'), + schema.literal('traces'), + schema.literal('synthetics'), + schema.literal('profiling'), + ]) + ), + datasetQuery: schema.maybe(schema.string()), + sortOrder: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { + defaultValue: 'asc', + }), + uncategorisedOnly: schema.boolean({ defaultValue: false }), + }), +}; + export const GetLimitedPackagesRequestSchema = { query: schema.object({ prerelease: schema.maybe(schema.boolean()), diff --git a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts index f3258e61d632f..06a67a13e425c 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/data_stream.ts @@ -106,6 +106,16 @@ export default function (providerContext: FtrProviderContext) { await uninstallPackage(pkgName, pkgUpdateVersion); }); + describe('Data Streams endpoint', () => { + it('Allows the fetching of data streams', async () => { + const res = await supertest + .get(`/api/fleet/epm/data_streams?uncategorisedOnly=false&datasetQuery=datastreams`) + .expect(200); + const dataStreams = res.body.items; + expect(dataStreams.length).to.be(6); + }); + }); + it('should list the logs and metrics datastream', async function () { await asyncForEach(namespaces, async (namespace) => { const resLogsDatastream = await es.transport.request( diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts index 71cd0f9d411d2..6924765458318 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/get.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts @@ -109,7 +109,63 @@ export default function (providerContext: FtrProviderContext) { expect(packageInfo.download).to.equal(undefined); await uninstallPackage(testPkgName, '9999.0.0'); }); + describe('Installed Packages', () => { + before(async () => { + await installPackage(testPkgName, testPkgVersion); + await installPackage('experimental', '0.1.0'); + await installPackage('endpoint', '8.6.1'); + }); + after(async () => { + await uninstallPackage(testPkgName, testPkgVersion); + await uninstallPackage('experimental', '0.1.0'); + await uninstallPackage('endpoint', '8.6.1'); + }); + it('Allows the fetching of installed packages', async () => { + const res = await supertest.get(`/api/fleet/epm/packages/installed`).expect(200); + const packages = res.body.items; + expect(packages.length).to.be(3); + }); + it('Can be limited with perPage', async () => { + const res = await supertest.get(`/api/fleet/epm/packages/installed?perPage=2`).expect(200); + const packages = res.body.items; + expect(packages.length).to.be(2); + }); + it('Can be queried by dataStreamType', async () => { + const res = await supertest + .get(`/api/fleet/epm/packages/installed?dataStreamType=metrics`) + .expect(200); + const packages = res.body.items; + let dataStreams = [] as any; + packages.forEach((packageItem: any) => { + dataStreams = dataStreams.concat(packageItem.dataStreams); + }); + const streamsWithWrongType = dataStreams.filter((stream: any) => { + return !stream.name.startsWith('metrics-'); + }); + expect(streamsWithWrongType.length).to.be(0); + }); + it('Can be sorted', async () => { + const ascRes = await supertest + .get(`/api/fleet/epm/packages/installed?sortOrder=asc`) + .expect(200); + const ascPackages = ascRes.body.items; + expect(ascPackages[0].name).to.be('apache'); + const descRes = await supertest + .get(`/api/fleet/epm/packages/installed?sortOrder=desc`) + .expect(200); + const descPackages = descRes.body.items; + expect(descPackages[0].name).to.be('experimental'); + }); + it('Can be filtered by name', async () => { + const res = await supertest + .get(`/api/fleet/epm/packages/installed?nameQuery=experimental`) + .expect(200); + const packages = res.body.items; + expect(packages.length).to.be(1); + expect(packages[0].name).to.be('experimental'); + }); + }); it('returns a 404 for a package that do not exists', async function () { await supertest.get('/api/fleet/epm/packages/notexists/99.99.99').expect(404); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/setup.ts b/x-pack/test/fleet_api_integration/apis/epm/setup.ts index f930166f4a74f..2d965682afa1e 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/setup.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/setup.ts @@ -18,10 +18,20 @@ export default function (providerContext: FtrProviderContext) { const log = getService('log'); const es = getService('es'); + const uninstallPackage = async (name: string, version: string) => { + await supertest + .delete(`/api/fleet/epm/packages/${name}/${version}`) + .set('kbn-xsrf', 'xxxx') + .send({ force: 'true' }); + }; + describe('setup api', async () => { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); - + after(async () => { + await uninstallPackage('deprecated', '0.1.0'); + await uninstallPackage('multiple_versions', '0.3.0'); + }); // FLAKY: https://github.com/elastic/kibana/issues/118479 describe.skip('setup performs upgrades', async () => { const oldEndpointVersion = '0.13.0';