From d6bf5bc04043f72b3dfa10016b0c91fc0cf24530 Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Fri, 26 Mar 2021 19:59:18 +0000 Subject: [PATCH 1/7] Update README with instructions for running with OpenSearch updated README with download and setup instructions for both OpenSearch and Dashboards. Closes #223 Signed-off-by: Bishoy Boktor --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 280bd7b25e83..780cdb2e69da 100644 --- a/README.md +++ b/README.md @@ -64,4 +64,4 @@ As we've been working, we've come up with a bunch of questions that we wanted to ### Questions? Feedback? -Let us know in the [forums](https://discuss.opendistrocommunity.dev/). w00t!!! \ No newline at end of file +Let us know in the [forums](https://discuss.opendistrocommunity.dev/). w00t!!! From 5bf0e7d7318ec98d57bad32290d223c014e96a8b Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Mon, 29 Mar 2021 21:22:48 +0000 Subject: [PATCH 2/7] Updates to allow Functional tests to run Fixed named references and switched default ftr run mode to source Signed-off-by: Bishoy Boktor --- packages/osd-opensearch/src/artifact.js | 8 ++++---- packages/osd-opensearch/src/cli_commands/source.js | 2 +- .../osd-test/src/functional_tests/lib/run_opensearch.js | 2 +- .../legacy_opensearch/legacy_opensearch_test_cluster.js | 5 +++-- plugins/.empty | 0 scripts/opensearch.js | 6 +++--- test/common/services/index.ts | 6 +++--- .../services/{legacy_es.ts => legacy_opensearch.ts} | 0 test/common/services/opensearch_archiver.ts | 2 +- .../common/services/opensearch_dashboards_server/index.ts | 2 +- .../apps/{timelion => timeline}/_expression_typeahead.js | 0 test/functional/apps/{timelion => timeline}/index.js | 0 12 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 plugins/.empty rename test/common/services/{legacy_es.ts => legacy_opensearch.ts} (100%) rename test/functional/apps/{timelion => timeline}/_expression_typeahead.js (100%) rename test/functional/apps/{timelion => timeline}/index.js (100%) diff --git a/packages/osd-opensearch/src/artifact.js b/packages/osd-opensearch/src/artifact.js index e5140a6203b6..ce4aa61d40ad 100644 --- a/packages/osd-opensearch/src/artifact.js +++ b/packages/osd-opensearch/src/artifact.js @@ -27,10 +27,10 @@ const { createHash } = require('crypto'); const path = require('path'); const asyncPipeline = promisify(pipeline); -const DAILY_SNAPSHOTS_BASE_URL = - 'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-daily'; -const PERMANENT_SNAPSHOTS_BASE_URL = - 'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-permanent'; +const DAILY_SNAPSHOTS_BASE_URL = ''; +//'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-daily'; +const PERMANENT_SNAPSHOTS_BASE_URL = ''; +//'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-permanent'; const { cache } = require('./utils'); const { resolveCustomSnapshotUrl } = require('./custom_snapshots'); diff --git a/packages/osd-opensearch/src/cli_commands/source.js b/packages/osd-opensearch/src/cli_commands/source.js index bd7cb5177c13..19b49531e126 100644 --- a/packages/osd-opensearch/src/cli_commands/source.js +++ b/packages/osd-opensearch/src/cli_commands/source.js @@ -24,7 +24,7 @@ const { Cluster } = require('../cluster'); exports.description = 'Build and run from source'; exports.help = (defaults = {}) => { - const { license = 'basic', password = 'changeme', 'base-path': basePath } = defaults; + const { license = 'oss', password = 'changeme', 'base-path': basePath } = defaults; return dedent` Options: diff --git a/packages/osd-test/src/functional_tests/lib/run_opensearch.js b/packages/osd-test/src/functional_tests/lib/run_opensearch.js index f78cdf3f68f7..c7e97da36d00 100644 --- a/packages/osd-test/src/functional_tests/lib/run_opensearch.js +++ b/packages/osd-test/src/functional_tests/lib/run_opensearch.js @@ -57,7 +57,7 @@ export async function runOpenSearch({ config, options }) { : config.get('servers.opensearch.password'), license, log, - basePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), + basePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), opensearchFrom: opensearchFrom || config.get('opensearchTestCluster.from'), dataArchive: config.get('opensearchTestCluster.dataArchive'), opensearchArgs, diff --git a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js index 8f074873615d..ee421566ddfc 100644 --- a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js +++ b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js @@ -35,7 +35,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { password = 'changeme', license = 'oss', log, - basePath = resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), + basePath = resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), opensearchFrom = opensearchTestConfig.getBuildFrom(), dataArchive, opensearchArgs: customOpenSearchArgs = [], @@ -58,7 +58,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { const config = { version: opensearchTestConfig.getVersion(), installPath: resolve(basePath, clusterName), - sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../opensearch'), + sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), password, license, basePath, @@ -78,6 +78,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { async start() { let installPath; + console.log('I"M RUNNING WOOO'); if (opensearchFrom === 'source') { installPath = (await cluster.installSource(config)).installPath; } else if (opensearchFrom === 'snapshot') { diff --git a/plugins/.empty b/plugins/.empty deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scripts/opensearch.js b/scripts/opensearch.js index 1da68c1e5160..80c050952bf0 100644 --- a/scripts/opensearch.js +++ b/scripts/opensearch.js @@ -25,11 +25,11 @@ var osdOpenSearch = require('@osd/opensearch'); osdOpenSearch .run({ - license: 'basic', + license: 'oss', password: 'changeme', version: pkg.version, - 'source-path': resolve(__dirname, '../../opensearch'), - 'base-path': resolve(__dirname, '../.opensearch'), + 'source-path': resolve(__dirname, '../../OpenSearch'), + 'base-path': resolve(__dirname, '../../OpenSearch'), ssl: false, }) .catch(function (e) { diff --git a/test/common/services/index.ts b/test/common/services/index.ts index d9863353b371..e733b117a6f4 100644 --- a/test/common/services/index.ts +++ b/test/common/services/index.ts @@ -19,9 +19,9 @@ import { DeploymentProvider } from './deployment'; import { LegacyOpenSearchProvider } from './legacy_opensearch'; -import { OpenSearchearchProvider } from './opensearch'; +import { OpenSearchProvider } from './opensearch'; import { OpenSearchArchiverProvider } from './opensearch_archiver'; -import { OpenSearchDashboardsServerProvider } from './opensearch_dasboards_server'; +import { OpenSearchDashboardsServerProvider } from './opensearch_dashboards_server'; import { RetryProvider } from './retry'; import { RandomnessProvider } from './randomness'; import { SecurityServiceProvider } from './security'; @@ -29,7 +29,7 @@ import { SecurityServiceProvider } from './security'; export const services = { deployment: DeploymentProvider, legacyOpenSearch: LegacyOpenSearchProvider, - opensearch: OpenSearchearchProvider, + opensearch: OpenSearchProvider, opensearchArchiver: OpenSearchArchiverProvider, opensearchDashboardsServer: OpenSearchDashboardsServerProvider, retry: RetryProvider, diff --git a/test/common/services/legacy_es.ts b/test/common/services/legacy_opensearch.ts similarity index 100% rename from test/common/services/legacy_es.ts rename to test/common/services/legacy_opensearch.ts diff --git a/test/common/services/opensearch_archiver.ts b/test/common/services/opensearch_archiver.ts index f83422ef769f..c84707e478cf 100644 --- a/test/common/services/opensearch_archiver.ts +++ b/test/common/services/opensearch_archiver.ts @@ -22,7 +22,7 @@ import { OpenSearchArchiver } from '@osd/opensearch-archiver'; import { FtrProviderContext } from '../ftr_provider_context'; // @ts-ignore not TS yet -import * as OpenSearchDashboardsServer from './opensearch_dasboards_server'; +import * as OpenSearchDashboardsServer from './opensearch_dashboards_server'; export function OpenSearchArchiverProvider({ getService, diff --git a/test/common/services/opensearch_dashboards_server/index.ts b/test/common/services/opensearch_dashboards_server/index.ts index 7326cf64bd66..23ba236b81fb 100644 --- a/test/common/services/opensearch_dashboards_server/index.ts +++ b/test/common/services/opensearch_dashboards_server/index.ts @@ -17,6 +17,6 @@ * under the License. */ -export { OpenSearchDashboardsServerProvider } from './opensearch_dasboards_server'; +export { OpenSearchDashboardsServerProvider } from './opensearch_dashboards_server'; // @ts-ignore export { extendOpenSearchArchiver } from './extend_opensearch_archiver'; diff --git a/test/functional/apps/timelion/_expression_typeahead.js b/test/functional/apps/timeline/_expression_typeahead.js similarity index 100% rename from test/functional/apps/timelion/_expression_typeahead.js rename to test/functional/apps/timeline/_expression_typeahead.js diff --git a/test/functional/apps/timelion/index.js b/test/functional/apps/timeline/index.js similarity index 100% rename from test/functional/apps/timelion/index.js rename to test/functional/apps/timeline/index.js From 5dab3159099d43f008417476119ce993cf4bf668 Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Fri, 2 Apr 2021 17:42:19 +0000 Subject: [PATCH 3/7] Address PR comments Signed-off-by: Bishoy Boktor --- packages/osd-opensearch/src/artifact.js | 2 -- packages/osd-test/src/functional_tests/lib/run_opensearch.js | 2 +- .../src/legacy_opensearch/legacy_opensearch_test_cluster.js | 3 +-- scripts/opensearch.js | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/osd-opensearch/src/artifact.js b/packages/osd-opensearch/src/artifact.js index ce4aa61d40ad..fd5007ec2bdc 100644 --- a/packages/osd-opensearch/src/artifact.js +++ b/packages/osd-opensearch/src/artifact.js @@ -28,9 +28,7 @@ const path = require('path'); const asyncPipeline = promisify(pipeline); const DAILY_SNAPSHOTS_BASE_URL = ''; -//'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-daily'; const PERMANENT_SNAPSHOTS_BASE_URL = ''; -//'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-permanent'; const { cache } = require('./utils'); const { resolveCustomSnapshotUrl } = require('./custom_snapshots'); diff --git a/packages/osd-test/src/functional_tests/lib/run_opensearch.js b/packages/osd-test/src/functional_tests/lib/run_opensearch.js index c7e97da36d00..f78cdf3f68f7 100644 --- a/packages/osd-test/src/functional_tests/lib/run_opensearch.js +++ b/packages/osd-test/src/functional_tests/lib/run_opensearch.js @@ -57,7 +57,7 @@ export async function runOpenSearch({ config, options }) { : config.get('servers.opensearch.password'), license, log, - basePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), + basePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), opensearchFrom: opensearchFrom || config.get('opensearchTestCluster.from'), dataArchive: config.get('opensearchTestCluster.dataArchive'), opensearchArgs, diff --git a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js index ee421566ddfc..6474786fec20 100644 --- a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js +++ b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js @@ -58,7 +58,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { const config = { version: opensearchTestConfig.getVersion(), installPath: resolve(basePath, clusterName), - sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), + sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), password, license, basePath, @@ -78,7 +78,6 @@ export function createLegacyOpenSearchTestCluster(options = {}) { async start() { let installPath; - console.log('I"M RUNNING WOOO'); if (opensearchFrom === 'source') { installPath = (await cluster.installSource(config)).installPath; } else if (opensearchFrom === 'snapshot') { diff --git a/scripts/opensearch.js b/scripts/opensearch.js index 80c050952bf0..29d991dc04a9 100644 --- a/scripts/opensearch.js +++ b/scripts/opensearch.js @@ -28,8 +28,8 @@ osdOpenSearch license: 'oss', password: 'changeme', version: pkg.version, - 'source-path': resolve(__dirname, '../../OpenSearch'), - 'base-path': resolve(__dirname, '../../OpenSearch'), + 'source-path': resolve(__dirname, '../../opensearch'), + 'base-path': resolve(__dirname, '../.opensearch'), ssl: false, }) .catch(function (e) { From 2ca9b8b6baadae299983c2acacc176ed53a42c6a Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Fri, 2 Apr 2021 17:45:03 +0000 Subject: [PATCH 4/7] Address PR comments Signed-off-by: Bishoy Boktor --- .../src/legacy_opensearch/legacy_opensearch_test_cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js index 6474786fec20..3775a04e11bc 100644 --- a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js +++ b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js @@ -35,7 +35,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { password = 'changeme', license = 'oss', log, - basePath = resolve(OPENSEARCH_DASHBOARDS_ROOT, '../OpenSearch'), + basePath = resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), opensearchFrom = opensearchTestConfig.getBuildFrom(), dataArchive, opensearchArgs: customOpenSearchArgs = [], From dee7b9fd8d6e5791fdbe80eb71b1b6664c7f962a Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Fri, 2 Apr 2021 18:55:08 +0000 Subject: [PATCH 5/7] Remove OPENSEARCH_TMPDIR check and security check from ftr run Removed the check for the self-created extraction directory and removed the no-longer existing security check arg Signed-off-by: Bishoy Boktor --- packages/osd-opensearch/src/cluster.js | 6 +++--- test/common/config.js | 2 +- test/security_functional/config.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js index 4577fa73bca9..8d0fdb5aa50a 100644 --- a/packages/osd-opensearch/src/cluster.js +++ b/packages/osd-opensearch/src/cluster.js @@ -291,9 +291,9 @@ exports.Cluster = class Cluster { this._process = execa(OPENSEARCH_BIN, args, { cwd: installPath, env: { - ...(installPath - ? { OPENSEARCH_TMPDIR: path.resolve(installPath, 'OPENSEARCH_TMPDIR') } - : {}), + //...(installPath + //? { OPENSEARCH_TMPDIR: path.resolve(installPath, 'OPENSEARCH_TMPDIR') } + //: {}), ...process.env, ...(options.bundledJDK ? { JAVA_HOME: '' } : {}), ...(options.opensearchEnvVars || {}), diff --git a/test/common/config.js b/test/common/config.js index 1e20033330ab..cde268bdfa32 100644 --- a/test/common/config.js +++ b/test/common/config.js @@ -50,7 +50,7 @@ export default function () { `--home.disableWelcomeScreen=true`, // Needed for async search functional tests to introduce a delay `--data.search.aggs.shardDelay.enabled=true`, - `--security.showInsecureClusterWarning=false`, + //`--security.showInsecureClusterWarning=false`, // '--telemetry.banner=false', // '--telemetry.optIn=false', // These are *very* important to have them pointing to staging diff --git a/test/security_functional/config.ts b/test/security_functional/config.ts index 8f141dd160b0..d7ac4b0197f3 100644 --- a/test/security_functional/config.ts +++ b/test/security_functional/config.ts @@ -45,7 +45,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { ...functionalConfig .get('osdTestServer.serverArgs') .filter((arg: string) => !arg.startsWith('--security.showInsecureClusterWarning')), - '--security.showInsecureClusterWarning=true', + // '--security.showInsecureClusterWarning=true', // Required to load new platform plugins via `--plugin-path` flag. '--env.name=development', ], From 5a7cda8bb822cfe04b2a973151407af4b189cc2d Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Fri, 2 Apr 2021 22:40:21 +0000 Subject: [PATCH 6/7] Update artifact unit tests Test Suites: 23 skipped, 1412 passed, 1412 of 1435 total Tests: 256 skipped, 9 todo, 10342 passed, 10607 total Snapshots: 2363 passed, 2363 total Time: 42.427 s, estimated 43 s Signed-off-by: Bishoy Boktor --- packages/osd-opensearch/src/artifact.test.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/osd-opensearch/src/artifact.test.js b/packages/osd-opensearch/src/artifact.test.js index e6a6ce8e4c66..a1116782efdf 100644 --- a/packages/osd-opensearch/src/artifact.test.js +++ b/packages/osd-opensearch/src/artifact.test.js @@ -33,10 +33,8 @@ const MOCK_VERSION = 'test-version'; const MOCK_URL = 'http://127.0.0.1:12345'; const MOCK_FILENAME = 'test-filename'; -const DAILY_SNAPSHOT_BASE_URL = - 'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-daily'; -const PERMANENT_SNAPSHOT_BASE_URL = - 'https://storage.googleapis.com/opensearch-dashboards-ci-opensearch-snapshots-permanent'; +const DAILY_SNAPSHOT_BASE_URL = ''; +const PERMANENT_SNAPSHOT_BASE_URL = ''; const createArchive = (params = {}) => { const license = params.license || 'default'; From 93e4276d3ad1edc9d65bf9638807f622a61a8141 Mon Sep 17 00:00:00 2001 From: Bishoy Boktor Date: Mon, 5 Apr 2021 17:35:02 +0000 Subject: [PATCH 7/7] Address PR comments Signed-off-by: Bishoy Boktor --- .../src/legacy_opensearch/legacy_opensearch_test_cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js index 3775a04e11bc..8f074873615d 100644 --- a/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js +++ b/packages/osd-test/src/legacy_opensearch/legacy_opensearch_test_cluster.js @@ -58,7 +58,7 @@ export function createLegacyOpenSearchTestCluster(options = {}) { const config = { version: opensearchTestConfig.getVersion(), installPath: resolve(basePath, clusterName), - sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '.opensearch'), + sourcePath: resolve(OPENSEARCH_DASHBOARDS_ROOT, '../opensearch'), password, license, basePath,