From d30e08759cc8c777b0b992c42710e67df86fe188 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 23 Apr 2020 16:11:55 -0700 Subject: [PATCH] Update tests, fixtures, and instructions --- x-pack/plugins/ingest_manager/README.md | 89 +++++++++++-------- .../api_integration/apis/fleet/agents/acks.ts | 6 +- .../apis/fleet/agents/actions.ts | 2 +- .../apis/fleet/agents/checkin.ts | 6 +- .../apis/fleet/agents/enroll.ts | 6 +- .../apis/fleet/unenroll_agent.ts | 10 +-- .../es_archives/fleet/agents/data.json | 66 +++++++------- .../es_archives/fleet/agents/mappings.json | 24 ++--- 8 files changed, 110 insertions(+), 99 deletions(-) diff --git a/x-pack/plugins/ingest_manager/README.md b/x-pack/plugins/ingest_manager/README.md index 9cd4821c2a727..0e7abcc3d74a9 100644 --- a/x-pack/plugins/ingest_manager/README.md +++ b/x-pack/plugins/ingest_manager/README.md @@ -1,60 +1,70 @@ # Ingest Manager + ## Plugin - - The plugin is disabled by default. See the TypeScript type for the [the available plugin configuration options](https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/common/types/index.ts#L9-L27) - - Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts) - - Adding `--xpack.ingestManager.epm.enabled=false` will disable the EPM API & UI - - Adding `--xpack.ingestManager.fleet.enabled=false` will disable the Fleet API & UI - - [code for adding the routes](https://github.com/elastic/kibana/blob/1f27d349533b1c2865c10c45b2cf705d7416fb36/x-pack/plugins/ingest_manager/server/plugin.ts#L115-L133) - - [Integration tests](server/integration_tests/router.test.ts) - - Both EPM and Fleet require `ingestManager` be enabled. They are not standalone features. + +- The plugin is disabled by default. See the TypeScript type for the [the available plugin configuration options](https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/common/types/index.ts#L9-L27) +- Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts) +- Adding `--xpack.ingestManager.epm.enabled=false` will disable the EPM API & UI +- Adding `--xpack.ingestManager.fleet.enabled=false` will disable the Fleet API & UI + - [code for adding the routes](https://github.com/elastic/kibana/blob/1f27d349533b1c2865c10c45b2cf705d7416fb36/x-pack/plugins/ingest_manager/server/plugin.ts#L115-L133) + - [Integration tests](server/integration_tests/router.test.ts) +- Both EPM and Fleet require `ingestManager` be enabled. They are not standalone features. ## Development ### Getting started -See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-kibana) +See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#running-kibana) One common development workflow is: - - Bootstrap Kibana - ``` - yarn kbn bootstrap - ``` - - Start Elasticsearch in one shell - ``` - yarn es snapshot -E xpack.security.authc.api_key.enabled=true - ``` - - Start Kibana in another shell - ``` - yarn start --xpack.ingestManager.enabled=true --no-base-path --xpack.endpoint.enabled=true - ``` + +- Bootstrap Kibana + ``` + yarn kbn bootstrap + ``` +- Start Elasticsearch in one shell + ``` + yarn es snapshot -E xpack.security.authc.api_key.enabled=true + ``` +- Start Kibana in another shell + ``` + yarn start --xpack.ingestManager.enabled=true --no-base-path --xpack.endpoint.enabled=true + ``` This plugin follows the `common`, `server`, `public` structure from the [Architecture Style Guide ](https://github.com/elastic/kibana/blob/master/style_guides/architecture_style_guide.md#file-and-folder-structure). We also follow the pattern of developing feature branches under your personal fork of Kibana. ### API Tests + #### Ingest & Fleet - 1. In one terminal, change to the `x-pack` directory and start the test server with - ``` - node scripts/functional_tests_server.js --config test/api_integration/config.ts - ``` - - 1. in a second terminal, run the tests from the Kibana root directory with - ``` - node scripts/functional_test_runner.js --config x-pack/test/api_integration/config.ts - ``` + +1. In one terminal, change to the `x-pack` directory and start the test server with + + ``` + node scripts/functional_tests_server.js --config test/api_integration/config.js + ``` + +1. in a second terminal, run the tests from the Kibana root directory with + ``` + node scripts/functional_test_runner.js --config x-pack/test/api_integration/config.js + ``` + #### EPM - 1. In one terminal, change to the `x-pack` directory and start the test server with - ``` - node scripts/functional_tests_server.js --config test/epm_api_integration/config.ts - ``` - 1. in a second terminal, run the tests from the Kibana root directory with - ``` - node scripts/functional_test_runner.js --config x-pack/test/epm_api_integration/config.ts - ``` +1. In one terminal, change to the `x-pack` directory and start the test server with + + ``` + node scripts/functional_tests_server.js --config test/epm_api_integration/config.ts + ``` - ### Staying up-to-date with `master` - While we're developing in the `feature-ingest` feature branch, here's is more information on keeping up to date with upstream kibana. +1. in a second terminal, run the tests from the Kibana root directory with + ``` + node scripts/functional_test_runner.js --config x-pack/test/epm_api_integration/config.ts + ``` + +### Staying up-to-date with `master` + +While we're developing in the `feature-ingest` feature branch, here's is more information on keeping up to date with upstream kibana.
merge upstream master into feature-ingest @@ -80,6 +90,7 @@ git push origin ## push your changes to upstream feature branch from the terminal; not GitHub UI git push upstream ``` +
See https://github.com/elastic/kibana/pull/37950 for an example. diff --git a/x-pack/test/api_integration/apis/fleet/agents/acks.ts b/x-pack/test/api_integration/apis/fleet/agents/acks.ts index f08ce33d8b60f..adde6dd184b81 100644 --- a/x-pack/test/api_integration/apis/fleet/agents/acks.ts +++ b/x-pack/test/api_integration/apis/fleet/agents/acks.ts @@ -32,12 +32,12 @@ export default function(providerContext: FtrProviderContext) { body: { _source: agentDoc }, } = await esClient.get({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', }); - agentDoc.agents.access_api_key_id = apiKey.id; + agentDoc['fleet-agents'].access_api_key_id = apiKey.id; await esClient.update({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', refresh: 'true', body: { doc: agentDoc, diff --git a/x-pack/test/api_integration/apis/fleet/agents/actions.ts b/x-pack/test/api_integration/apis/fleet/agents/actions.ts index cf0641acf9e1c..577299e652610 100644 --- a/x-pack/test/api_integration/apis/fleet/agents/actions.ts +++ b/x-pack/test/api_integration/apis/fleet/agents/actions.ts @@ -67,7 +67,7 @@ export default function(providerContext: FtrProviderContext) { }, }) .expect(404); - expect(apiResponse.message).to.eql('Saved object [agents/agent100] not found'); + expect(apiResponse.message).to.eql('Saved object [fleet-agents/agent100] not found'); }); }); } diff --git a/x-pack/test/api_integration/apis/fleet/agents/checkin.ts b/x-pack/test/api_integration/apis/fleet/agents/checkin.ts index ca51676126e73..b405b5065bc0e 100644 --- a/x-pack/test/api_integration/apis/fleet/agents/checkin.ts +++ b/x-pack/test/api_integration/apis/fleet/agents/checkin.ts @@ -32,12 +32,12 @@ export default function(providerContext: FtrProviderContext) { body: { _source: agentDoc }, } = await esClient.get({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', }); - agentDoc.agents.access_api_key_id = apiKey.id; + agentDoc['fleet-agents'].access_api_key_id = apiKey.id; await esClient.update({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', refresh: 'true', body: { doc: agentDoc, diff --git a/x-pack/test/api_integration/apis/fleet/agents/enroll.ts b/x-pack/test/api_integration/apis/fleet/agents/enroll.ts index d8e9749744ea4..c934ddf8a406b 100644 --- a/x-pack/test/api_integration/apis/fleet/agents/enroll.ts +++ b/x-pack/test/api_integration/apis/fleet/agents/enroll.ts @@ -33,13 +33,13 @@ export default function(providerContext: FtrProviderContext) { body: { _source: enrollmentApiKeyDoc }, } = await esClient.get({ index: '.kibana', - id: 'enrollment_api_keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0', + id: 'fleet-enrollment-api-keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0', }); // @ts-ignore - enrollmentApiKeyDoc.enrollment_api_keys.api_key_id = apiKey.id; + enrollmentApiKeyDoc['fleet-enrollment-api-keys'].api_key_id = apiKey.id; await esClient.update({ index: '.kibana', - id: 'enrollment_api_keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0', + id: 'fleet-enrollment-api-keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0', refresh: 'true', body: { doc: enrollmentApiKeyDoc, diff --git a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts index d33b92acf95a5..5b8e03269ceef 100644 --- a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts +++ b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts @@ -40,18 +40,18 @@ export default function(providerContext: FtrProviderContext) { body: { _source: agentDoc }, } = await esClient.get({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', }); // @ts-ignore - agentDoc.agents.access_api_key_id = accessAPIKeyId; - agentDoc.agents.default_api_key_id = outputAPIKeyBody.id; - agentDoc.agents.default_api_key = Buffer.from( + agentDoc['fleet-agents'].access_api_key_id = accessAPIKeyId; + agentDoc['fleet-agents'].default_api_key_id = outputAPIKeyBody.id; + agentDoc['fleet-agents'].default_api_key = Buffer.from( `${outputAPIKeyBody.id}:${outputAPIKeyBody.api_key}` ).toString('base64'); await esClient.update({ index: '.kibana', - id: 'agents:agent1', + id: 'fleet-agents:agent1', refresh: 'true', body: { doc: agentDoc, diff --git a/x-pack/test/functional/es_archives/fleet/agents/data.json b/x-pack/test/functional/es_archives/fleet/agents/data.json index 1ffb119ca1023..3fe4f828ba128 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/data.json +++ b/x-pack/test/functional/es_archives/fleet/agents/data.json @@ -1,11 +1,11 @@ { "type": "doc", "value": { - "id": "agents:agent1", + "id": "fleet-agents:agent1", "index": ".kibana", "source": { - "type": "agents", - "agents": { + "type": "fleet-agents", + "fleet-agents": { "access_api_key_id": "api-key-2", "active": true, "shared_id": "agent1_filebeat", @@ -21,11 +21,11 @@ { "type": "doc", "value": { - "id": "agents:agent2", + "id": "fleet-agents:agent2", "index": ".kibana", "source": { - "type": "agents", - "agents": { + "type": "fleet-agents", + "fleet-agents": { "access_api_key_id": "api-key-2", "active": true, "shared_id": "agent2_filebeat", @@ -40,11 +40,11 @@ { "type": "doc", "value": { - "id": "agents:agent3", + "id": "fleet-agents:agent3", "index": ".kibana", "source": { - "type": "agents", - "agents": { + "type": "fleet-agents", + "fleet-agents": { "access_api_key_id": "api-key-3", "active": true, "shared_id": "agent3_metricbeat", @@ -59,11 +59,11 @@ { "type": "doc", "value": { - "id": "agents:agent4", + "id": "fleet-agents:agent4", "index": ".kibana", "source": { - "type": "agents", - "agents": { + "type": "fleet-agents", + "fleet-agents": { "access_api_key_id": "api-key-4", "active": true, "shared_id": "agent4_metricbeat", @@ -78,17 +78,17 @@ { "type": "doc", "value": { - "id": "enrollment_api_keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0", + "id": "fleet-enrollment-api-keys:ed22ca17-e178-4cfe-8b02-54ea29fbd6d0", "index": ".kibana", "source": { - "enrollment_api_keys" : { + "fleet-enrollment-api-keys" : { "created_at" : "2019-10-10T16:31:12.518Z", "name": "FleetEnrollmentKey:1", "api_key_id" : "key", "config_id" : "policy:1", "active" : true }, - "type" : "enrollment_api_keys", + "type" : "fleet-enrollment-api-keys", "references": [] } } @@ -97,11 +97,11 @@ { "type": "doc", "value": { - "id": "events:event1", + "id": "fleet-agent-events:event1", "index": ".kibana", "source": { - "type": "agent_events", - "agent_events": { + "type": "fleet-agent-events", + "fleet-agent-events": { "agent_id": "agent1", "type": "STATE", "subtype": "STARTED", @@ -116,11 +116,11 @@ { "type": "doc", "value": { - "id": "events:event2", + "id": "fleet-agent-events:event2", "index": ".kibana", "source": { - "type": "agent_events", - "agent_events": { + "type": "fleet-agent-events", + "fleet-agent-events": { "agent_id": "agent1", "type": "STATE", "subtype": "STOPPED", @@ -135,11 +135,11 @@ { "type": "doc", "value": { - "id": "agent_actions:37ed51ff-e80f-4f2a-a62d-f4fa975e7d85", + "id": "fleet-agent-actions:37ed51ff-e80f-4f2a-a62d-f4fa975e7d85", "index": ".kibana", "source": { - "type": "agent_actions", - "agent_actions": { + "type": "fleet-agent-actions", + "fleet-agent-actions": { "agent_id": "agent1", "created_at": "2019-09-04T15:04:07+0000", "type": "RESUME", @@ -152,11 +152,11 @@ { "type": "doc", "value": { - "id": "agent_actions:b400439c-bbbf-43d5-83cb-cf8b7e32506f", + "id": "fleet-agent-actions:b400439c-bbbf-43d5-83cb-cf8b7e32506f", "index": ".kibana", "source": { - "type": "agent_actions", - "agent_actions": { + "type": "fleet-agent-actions", + "fleet-agent-actions": { "agent_id": "agent1", "type": "PAUSE", "created_at": "2019-09-04T15:01:07+0000", @@ -169,11 +169,11 @@ { "type": "doc", "value": { - "id": "agent_actions:48cebde1-c906-4893-b89f-595d943b72a1", + "id": "fleet-agent-actions:48cebde1-c906-4893-b89f-595d943b72a1", "index": ".kibana", "source": { - "type": "agent_actions", - "agent_actions": { + "type": "fleet-agent-actions", + "fleet-agent-actions": { "agent_id": "agent1", "type": "CONFIG_CHANGE", "created_at": "2020-03-15T03:47:15.129Z", @@ -186,11 +186,11 @@ { "type": "doc", "value": { - "id": "agent_actions:48cebde1-c906-4893-b89f-595d943b72a2", + "id": "fleet-agent-actions:48cebde1-c906-4893-b89f-595d943b72a2", "index": ".kibana", "source": { - "type": "agent_actions", - "agent_actions": { + "type": "fleet-agent-actions", + "fleet-agent-actions": { "agent_id": "agent1", "type": "CONFIG_CHANGE", "created_at": "2020-03-15T03:47:15.129Z", diff --git a/x-pack/test/functional/es_archives/fleet/agents/mappings.json b/x-pack/test/functional/es_archives/fleet/agents/mappings.json index 31ae161049303..5d5d373797d4c 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/mappings.json +++ b/x-pack/test/functional/es_archives/fleet/agents/mappings.json @@ -9,7 +9,7 @@ "dynamic": "strict", "_meta": { "migrationMappingPropertyHashes": { - "outputs": "aee9782e0d500b867859650a36280165", + "ingest-outputs": "aee9782e0d500b867859650a36280165", "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", "visualization": "52d7a13ad68a150c4525b292d23e12cc", "references": "7997cf5a56cc02bdc9c93361bde732b0", @@ -23,14 +23,14 @@ "dashboard": "d00f614b29a80360e1190193fd333bab", "metrics-explorer-view": "53c5365793677328df0ccb6138bf3cdd", "siem-detection-engine-rule-actions": "90eee2e4635260f4be0a1da8f5bc0aa0", - "agent_events": "3231653fafe4ef3196fe3b32ab774bf2", + "fleet-agent-events": "3231653fafe4ef3196fe3b32ab774bf2", "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", "application_usage_transactional": "965839e75f809fefe04f92dc4d99722a", "action_task_params": "a9d49f184ee89641044be0ca2950fa3a", "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd", "inventory-view": "9ecce5b58867403613d82fe496470b34", - "enrollment_api_keys": "28b91e20b105b6f928e2012600085d8f", + "fleet-enrollment-api-keys": "28b91e20b105b6f928e2012600085d8f", "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6", "cases-comments": "c2061fb929f585df57425102fa928b4b", "canvas-element": "7390014e1091044523666d97247392fc", @@ -50,20 +50,20 @@ "siem-detection-engine-rule-status": "ae783f41c6937db6b7a2ef5c93a9e9b0", "map": "23d7aa4a720d4938ccde3983f87bd58d", "uptime-dynamic-settings": "b6289473c8985c79b6c47eebc19a0ca5", - "epm-package": "75d12cd13c867fd713d7dfb27366bc20", + "epm-packages": "75d12cd13c867fd713d7dfb27366bc20", "apm-telemetry": "3525d7c22c42bc80f5e6e9cb3f2b26a2", "cases": "08b8b110dbca273d37e8aef131ecab61", "siem-ui-timeline": "ac8020190f5950dd3250b6499144e7fb", "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", "url": "c7f66a0df8b1b52f17c28c4adb111105", - "agents": "c3eeb7b9d97176f15f6d126370ab23c7", + "fleet-agents": "c3eeb7b9d97176f15f6d126370ab23c7", "migrationVersion": "4a1746014a75ade3a714e1db5763276f", "index-pattern": "66eccb05066c5a89924f48a9e9736499", "maps-telemetry": "268da3a48066123fc5baf35abaa55014", "namespace": "2f4316de49999235636386fe51dc06c1", "cases-user-actions": "32277330ec6b721abe3b846cfd939a71", - "agent_actions": "ed270b46812f0fa1439366c428a2cf17", + "fleet-agent-actions": "ed270b46812f0fa1439366c428a2cf17", "siem-ui-timeline-pinned-event": "20638091112f0e14f0e443d512301c29", "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", "config": "ae24d22d5986d04124cc6568f771066f", @@ -107,7 +107,7 @@ } } }, - "agent_actions": { + "fleet-agent-actions": { "properties": { "agent_id": { "type": "keyword" @@ -160,7 +160,7 @@ } } }, - "agent_events": { + "fleet-agent-events": { "properties": { "action_id": { "type": "keyword" @@ -194,7 +194,7 @@ } } }, - "agents": { + "fleet-agents": { "properties": { "access_api_key_id": { "type": "keyword" @@ -1705,7 +1705,7 @@ } } }, - "enrollment_api_keys": { + "fleet-enrollment-api-keys": { "properties": { "active": { "type": "boolean" @@ -1736,7 +1736,7 @@ } } }, - "epm-package": { + "epm-packages": { "properties": { "installed": { "type": "nested", @@ -2211,7 +2211,7 @@ "namespace": { "type": "keyword" }, - "outputs": { + "ingest-outputs": { "properties": { "api_key": { "type": "keyword"