Skip to content

Commit

Permalink
[Fleet] Rename ingest_manager_api_integration tests fleet_api_integra…
Browse files Browse the repository at this point in the history
…tion (#83011) (#83326)
  • Loading branch information
nchaulet authored Nov 12, 2020
1 parent 5f36a27 commit acd3e35
Show file tree
Hide file tree
Showing 168 changed files with 107 additions and 143 deletions.
2 changes: 1 addition & 1 deletion src/dev/precommit_hook/casing_check_config.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ export const IGNORE_FILE_GLOBS = [

'x-pack/plugins/maps/server/fonts/**/*',
// packages for the ingest manager's api integration tests could be valid semver which has dashes
'x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/**/*',
'x-pack/test/fleet_api_integration/apis/fixtures/test_packages/**/*',
];

/**
Expand Down
4 changes: 2 additions & 2 deletions vars/kibanaPipeline.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
def kibanaPort = "61${parallelId}1"
def esPort = "61${parallelId}2"
def esTransportPort = "61${parallelId}3"
def ingestManagementPackageRegistryPort = "61${parallelId}4"
def fleetPackageRegistryPort = "61${parallelId}4"
def alertingProxyPort = "61${parallelId}5"

withEnv([
Expand All @@ -100,7 +100,7 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
"TEST_ES_URL=http://elastic:changeme@localhost:${esPort}",
"TEST_ES_TRANSPORT_PORT=${esTransportPort}",
"KBN_NP_PLUGINS_BUILT=true",
"INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}",
"FLEET_PACKAGE_REGISTRY_PORT=${fleetPackageRegistryPort}",
"ALERTING_PROXY_PORT=${alertingProxyPort}"
] + additionalEnvs) {
closure()
Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugins/fleet/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,17 @@ You need to have `docker` to run ingest manager api integration tests
1. In one terminal, run the tests from the Kibana root directory with

```
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/ingest_manager_api_integration/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/fleet_api_integration/config.ts
```

1. in a second terminal, run the tests from the Kibana root directory with

```
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/ingest_manager_api_integration/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/fleet_api_integration/config.ts
```

Optionally you can filter which tests you want to run using `--grep`

```
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/ingest_manager_api_integration/config.ts --grep='fleet'
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/fleet_api_integration/config.ts --grep='fleet'
```
24 changes: 12 additions & 12 deletions x-pack/plugins/fleet/dev_docs/api_integration_tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,29 @@ Many API integration tests for Ingest Manager trigger at some point a connection
- the deployed registry is temporarily unavailable
- the packages served by the registry do not match the expectation of the code under test

For that reason, we run a dockerized version of the package registry in Kibana CI. For this to work, our tests must run against a custom test configuration and be kept in a custom directory, `x-pack/test/ingest_manager_api_integration`.
For that reason, we run a dockerized version of the package registry in Kibana CI. For this to work, our tests must run against a custom test configuration and be kept in a custom directory, `x-pack/test/fleet_api_integration`.

## How to run the tests locally

Usually, having the test server and the test runner in two different shells is most efficient, as it is possible to keep the server running and only rerun the test runner as often as needed. To do so, in one shell in the main `kibana` directory, run:

```
$ export INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr:server --config x-pack/test/ingest_manager_api_integration/config.ts
$ export FLEET_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr:server --config x-pack/test/fleet_api_integration/config.ts
```

In another shell in the same directory, run

```
$ export INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr:runner --config x-pack/test/ingest_manager_api_integration/config.ts
$ export FLEET_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr:runner --config x-pack/test/fleet_api_integration/config.ts
```

However, it is also possible to **alternatively** run everything in one go, again from the main `kibana` directory:

```
$ export INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr --config x-pack/test/ingest_manager_api_integration/config.ts
$ export FLEET_PACKAGE_REGISTRY_PORT=12345
$ yarn test:ftr --config x-pack/test/fleet_api_integration/config.ts
```

Port `12345` is used as an example here, it can be anything, but the environment variable has to be present for the tests to run at all.
Expand All @@ -37,7 +37,7 @@ Port `12345` is used as an example here, it can be anything, but the environment
We use the `DockerServers` service provided by `kbn-test`. The documentation for this functionality can be found here:
https://github.com/elastic/kibana/blob/master/packages/kbn-test/src/functional_test_runner/lib/docker_servers/README.md

The main configuration for the `DockerServers` service for our tests can be found in `x-pack/test/ingest_manager_api_integration/config.ts`:
The main configuration for the `DockerServers` service for our tests can be found in `x-pack/test/fleet_api_integration/config.ts`:

### Specify the arguments to pass to `docker run`:

Expand Down Expand Up @@ -74,13 +74,13 @@ The containerized package registry contains a set of packages which should be su
docker run -p 8080:8080 docker.elastic.co/package-registry/package-registry:kibana-testing-1
```

Additional packages for testing certain corner cases or error conditions can be put into `x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages`. A package `filetest` has been added there as an example.
Additional packages for testing certain corner cases or error conditions can be put into `x-pack/test/fleet_api_integration/apis/fixtures/test_packages`. A package `filetest` has been added there as an example.

## Some DockerServers background

For the `DockerServers` servers to run correctly in CI, the `INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT` environment variable needs to be under control of the CI environment. The reason behind this: it is possible that several versions of our tests are run in parallel on the same worker in Jenkins, and if we used a hard-coded port number here, those tests would run into port conflicts. (This is also the case for a few other ports, and the setup happens in `vars/kibanaPipeline.groovy`).
For the `DockerServers` servers to run correctly in CI, the `FLEET_PACKAGE_REGISTRY_PORT` environment variable needs to be under control of the CI environment. The reason behind this: it is possible that several versions of our tests are run in parallel on the same worker in Jenkins, and if we used a hard-coded port number here, those tests would run into port conflicts. (This is also the case for a few other ports, and the setup happens in `vars/kibanaPipeline.groovy`).

Also, not every developer has `docker` installed on their workstation, so it must be possible to run the testsuite as a whole without `docker`, and preferably this should be the default behaviour. Therefore, our `DockerServers` service is only enabled when `INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT` is set. This needs to be checked in every test like this:
Also, not every developer has `docker` installed on their workstation, so it must be possible to run the testsuite as a whole without `docker`, and preferably this should be the default behaviour. Therefore, our `DockerServers` service is only enabled when `FLEET_PACKAGE_REGISTRY_PORT` is set. This needs to be checked in every test like this:

```
it('fetches a .json search file', async function () {
Expand All @@ -105,7 +105,7 @@ If the tests are skipped in this way, they are marked in the test summary as `pe
└-> "before all" hook
└-> lists all packages from the registry
└-> "before each" hook: global before each
│ warn disabling tests because DockerServers service is not enabled, set INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT to run them
│ warn disabling tests because DockerServers service is not enabled, set FLEET_PACKAGE_REGISTRY_PORT to run them
└-> lists all packages from the registry
└-> "after all" hook
[...]
Expand Down
10 changes: 5 additions & 5 deletions x-pack/plugins/security_solution/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ The endpoint functional tests are located [here](../../test/security_solution_en
### Using Docker

To run the tests using the recommended docker image version you must have `docker` installed. The testing infrastructure
will stand up a docker container using the image defined [here](../../test/ingest_manager_api_integration/config.ts#L15)
will stand up a docker container using the image defined [here](../../test/fleet_api_integration/config.ts#L15)

Make sure you're in the Kibana root directory.

Expand All @@ -37,27 +37,27 @@ Make sure you're in the Kibana root directory.
In one terminal, run:

```bash
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
```

In another terminal, run:

```bash
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
```

#### Endpoint Functional Tests

In one terminal, run:

```bash
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
```

In another terminal, run:

```bash
INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
```

### Running your own package registry
Expand Down
2 changes: 1 addition & 1 deletion x-pack/scripts/functional_tests.js
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ require('@kbn/test').runTestsCli([
require.resolve('../test/reporting_api_integration/reporting_and_security.config.ts'),
require.resolve('../test/reporting_api_integration/reporting_without_security.config.ts'),
require.resolve('../test/security_solution_endpoint_api_int/config.ts'),
require.resolve('../test/ingest_manager_api_integration/config.ts'),
require.resolve('../test/fleet_api_integration/config.ts'),
require.resolve('../test/functional_enterprise_search/without_host_configured.config.ts'),
require.resolve('../test/functional_vis_wizard/config.ts'),
require.resolve('../test/saved_object_tagging/functional/config.ts'),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');

describe('ingest_manager_agent_policies', () => {
describe('fleet_agent_policies', () => {
const createdPolicyIds: string[] = [];
after(async () => {
const deletedPromises = createdPolicyIds.map((agentPolicyId) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import expect from '@kbn/expect';
import { skipIfNoDockerRegistry } from '../../helpers';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { setupIngest, getSupertestWithoutAuth } from '../fleet/agents/services';
import { setupFleetAndAgents, getSupertestWithoutAuth } from '../agents/services';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand Down Expand Up @@ -68,7 +68,7 @@ export default function (providerContext: FtrProviderContext) {
}

// Test all the side effect that should occurs when we create|update an agent policy
describe('ingest_manager_agent_policies_with_agents_setup', () => {
describe('fleet_agent_policies_with_agents_setup', () => {
skipIfNoDockerRegistry(providerContext);

before(async () => {
Expand All @@ -78,7 +78,7 @@ export default function (providerContext: FtrProviderContext) {
await esArchiver.unload('fleet/agents');
});

setupIngest(providerContext);
setupFleetAndAgents(providerContext);

describe('POST /api/fleet/agent_policies', () => {
it('should create an enrollment key and an agent action `POLICY_CHANGE` for the policy', async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/

export default function loadTests({ loadTestFile }) {
describe('Ingest Manager Endpoints', () => {
describe('Fleet Endpoints', () => {
loadTestFile(require.resolve('./agent_policy_with_agents_setup'));
loadTestFile(require.resolve('./agent_policy'));
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import expect from '@kbn/expect';
import uuid from 'uuid';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { getSupertestWithoutAuth } from './services';

export default function (providerContext: FtrProviderContext) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/

import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
import expect from '@kbn/expect';
import uuid from 'uuid';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { getSupertestWithoutAuth, setupIngest } from './services';
import { skipIfNoDockerRegistry } from '../../../helpers';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { getSupertestWithoutAuth, setupFleetAndAgents } from './services';
import { skipIfNoDockerRegistry } from '../../helpers';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand Down Expand Up @@ -46,7 +46,7 @@ export default function (providerContext: FtrProviderContext) {
},
});
});
setupIngest(providerContext);
setupFleetAndAgents(providerContext);
after(async () => {
await esArchiver.unload('fleet/agents');
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

import expect from '@kbn/expect';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { setupIngest, getSupertestWithoutAuth } from './services';
import { skipIfNoDockerRegistry } from '../../../helpers';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { setupFleetAndAgents, getSupertestWithoutAuth } from './services';
import { skipIfNoDockerRegistry } from '../../helpers';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand All @@ -24,7 +24,7 @@ export default function (providerContext: FtrProviderContext) {
before(async () => {
await esArchiver.load('empty_kibana');
});
setupIngest(providerContext);
setupFleetAndAgents(providerContext);
after(async () => {
await esArchiver.unload('empty_kibana');
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/

import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';

export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
import expect from '@kbn/expect';
import uuid from 'uuid';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { getSupertestWithoutAuth, setupIngest, getEsClientForAPIKey } from './services';
import { skipIfNoDockerRegistry } from '../../../helpers';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { getSupertestWithoutAuth, setupFleetAndAgents, getEsClientForAPIKey } from './services';
import { skipIfNoDockerRegistry } from '../../helpers';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand Down Expand Up @@ -52,7 +52,7 @@ export default function (providerContext: FtrProviderContext) {
const kibanaVersionAccessor = kibanaServer.version;
kibanaVersion = await kibanaVersionAccessor.get();
});
setupIngest(providerContext);
setupFleetAndAgents(providerContext);
after(async () => {
await esArchiver.unload('fleet/agents');
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import expect from '@kbn/expect';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';

export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import expect from '@kbn/expect';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';

export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { setupIngest } from './services';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { setupFleetAndAgents } from './services';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');

describe('fleet_reassign_agent', () => {
setupIngest(providerContext);
setupFleetAndAgents(providerContext);
before(async () => {
await esArchiver.loadIfNeeded('fleet/agents');
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import supertestAsPromised from 'supertest-as-promised';
import { Client } from '@elastic/elasticsearch';
import { format as formatUrl } from 'url';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';

export function getSupertestWithoutAuth({ getService }: FtrProviderContext) {
const config = getService('config');
Expand All @@ -31,7 +31,7 @@ export function getEsClientForAPIKey({ getService }: FtrProviderContext, esApiKe
});
}

export function setupIngest({ getService }: FtrProviderContext) {
export function setupFleetAndAgents({ getService }: FtrProviderContext) {
before(async () => {
await getService('supertest').post(`/api/fleet/setup`).set('kbn-xsrf', 'xxx').send();
await getService('supertest')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
import expect from '@kbn/expect';
import uuid from 'uuid';

import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { setupIngest } from './services';
import { skipIfNoDockerRegistry } from '../../../helpers';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { setupFleetAndAgents } from './services';
import { skipIfNoDockerRegistry } from '../../helpers';

export default function (providerContext: FtrProviderContext) {
const { getService } = providerContext;
Expand All @@ -24,7 +24,7 @@ export default function (providerContext: FtrProviderContext) {
before(async () => {
await esArchiver.loadIfNeeded('fleet/agents');
});
setupIngest(providerContext);
setupFleetAndAgents(providerContext);
beforeEach(async () => {
const { body: accessAPIKeyBody } = await esClient.security.createApiKey({
body: {
Expand Down
Loading

0 comments on commit acd3e35

Please sign in to comment.