Merge branch '2.x' into chore/2.x-vega-lite-v5-from-source

.github/workflows/build_and_test_workflow.yml

@@ -100,8 +100,13 @@ jobs:
           npm uninstall -g yarn
           npm i -g [email protected]
 
+      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
+      # Increase network timeout for Windows, retry once if bootstrap fails
       - name: Run bootstrap
-        run: yarn osd bootstrap
+        run: | 
+          yarn cache clean
+          yarn config set network-timeout 1000000 -g
+          yarn osd bootstrap || yarn osd bootstrap
 
       - name: Run linter
         id: linter
@@ -152,9 +157,8 @@ jobs:
           npm uninstall -g yarn
           npm i -g [email protected]
 
-      # image has the latest chrome v99
       - name: Setup chromedriver
-        run: yarn add --dev [email protected]
+        run: node scripts/upgrade_chromedriver.js
 
       - name: Run bootstrap
         run: yarn osd bootstrap
@@ -204,12 +208,16 @@ jobs:
           npm uninstall -g yarn
           npm i -g [email protected]
 
-      # image has the latest chrome v99
       - name: Setup chromedriver
-        run: yarn add --dev [email protected]
+        run: node scripts/upgrade_chromedriver.js
 
+      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
+      # Increase network timeout for Windows, retry once if bootstrap fails
       - name: Run bootstrap
-        run: yarn osd bootstrap
+        run: | 
+          yarn cache clean
+          yarn config set network-timeout 1000000 -g
+          yarn osd bootstrap || yarn osd bootstrap
 
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10

.gitignore

@@ -3,6 +3,7 @@
 .ackrc
 /.opensearch
 /.chromium
+/package.json.bak
 .DS_Store
 .node_binaries
 .native_modules

CHANGELOG.md

@@ -37,6 +37,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Save Object Aggregation View] Fix for export all after scroll count response changed in PR#2656 ([#2696](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2696))
 
 ### 🐛 Bug Fixes
+
 * [Vis Builder] Fixes auto bounds for timeseries bar chart visualization ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))
 * [Vis Builder] Fixes visualization shift when editing agg ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))
 * [Vis Builder] Renames "Histogram" to "Bar" in vis type picker ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))
@@ -49,11 +50,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 * [Multi DataSource] Apply get indices error handling in step index pattern ([#2652](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2652))
 * Removed Leftover X Pack references ([#2638](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2638))
 * Bumped `del` version to fix MacOS race condition ([#2847](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2873))
+* [CI] Update test workflow to increase network-timeout for yarn for installing dependencies ([#3118](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3118))
 
 ### 🚞 Infrastructure
 
 - Add CHANGELOG.md and related workflows ([#2414](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2414))
 - Update backport custom branch name to utilize head template ([#2766](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2766))
+- Add automatic selection of the appropriate version of chrome driver to run functional tests ([#2990](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2990))
 
 ### 📝 Documentation
 

DEVELOPER_GUIDE.md

@@ -36,6 +36,11 @@ $ npm i -g yarn
 $ yarn osd bootstrap # This command will also install npm dependencies
 ```
 
+If you experience a network timeout while bootstrapping, you can update the timeout by configuring it in the `.yarnrc`. For example:
+```
+network-timeout 1000000
+```
+
 ### Configure OpenSearch Dashboards
 
 *This step is only mandatory if you have https/authentication enabled, or if you use the OpenSearch Docker image in its default configuration.*

package.json

@@ -264,6 +264,7 @@
     "@types/dedent": "^0.7.0",
     "@types/deep-freeze-strict": "^1.1.0",
     "@types/delete-empty": "^2.0.0",
+    "@types/dompurify": "^2.3.3",
     "@types/elasticsearch": "^5.0.33",
     "@types/enzyme": "^3.10.7",
     "@types/eslint": "^6.1.3",
@@ -352,6 +353,7 @@
     "d3-cloud": "1.2.5",
     "dedent": "^0.7.0",
     "delete-empty": "^2.0.0",
+    "dompurify": "^2.4.1",
     "enzyme": "^3.11.0",
     "enzyme-adapter-react-16": "^1.15.5",
     "enzyme-to-json": "^3.5.0",

packages/osd-optimizer/src/worker/run_worker.ts

@@ -78,11 +78,13 @@ const exit = (code: number) => {
   setTimeout(() => {
     send(
       workerMsgs.error(
-        new Error('process did not automatically exit within 5 seconds, forcing exit')
+        new Error(
+          `process did not automatically exit within 15 seconds (previous code: ${code}); forcing exit...`
+        )
       )
     );
     process.exit(1);
-  }, 5000).unref();
+  }, 15000).unref();
 };
 
 // check for connected parent on an unref'd timer rather than listening

scripts/upgrade_chromedriver.js

@@ -0,0 +1,122 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Upgrades the chromedriver dev-dependency to the one supported by the version of Google Chrome
+ * installed on the machine.
+ *
+ * Usage: node scripts/upgrade_chromedriver.js [--install]
+ */
+
+/* eslint no-restricted-syntax: 0 */
+const { execSync, spawnSync } = require('child_process');
+const { createReadStream, createWriteStream, unlinkSync, renameSync, existsSync } = require('fs');
+const { createInterface } = require('readline');
+
+if (!process.argv.includes(__filename)) {
+  console.error('Usage: node scripts/upgrade_chromedriver.js [--install]');
+  process.exit(1);
+}
+
+const versionCheckCommands = [];
+
+switch (process.platform) {
+  case 'win32':
+    versionCheckCommands.push(
+      'powershell "(Get-Item \\"$Env:Programfiles/Google/Chrome/Application/chrome.exe\\").VersionInfo.FileVersion"'
+    );
+    break;
+
+  case 'darwin':
+    versionCheckCommands.push(
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --version'
+    );
+    break;
+
+  default:
+    versionCheckCommands.push(
+      ...[
+        '/usr/bin',
+        '/usr/local/bin',
+        '/usr/sbin',
+        '/usr/local/sbin',
+        '/opt/bin',
+        '/usr/bin/X11',
+        '/usr/X11R6/bin',
+      ].flatMap((loc) =>
+        [
+          'google-chrome --version',
+          'google-chrome-stable --version',
+          'chromium --version',
+          'chromium-browser --version',
+        ].map((cmd) => `${loc}/${cmd}`)
+      )
+    );
+}
+
+let versionCheckOutput;
+versionCheckCommands.some((cmd) => {
+  try {
+    console.log(cmd);
+    versionCheckOutput = execSync(cmd, { encoding: 'utf8' })?.trim?.();
+    return true;
+  } catch (e) {
+    console.log('Failed to get version using', cmd);
+  }
+});
+
+// Versions 90+
+const majorVersion = versionCheckOutput?.match?.(/(?:^|\s)(9\d|\d{3})\./)?.[1];
+
+if (majorVersion) {
+  if (process.argv.includes('--install')) {
+    console.log(`Installing chromedriver@^${majorVersion}`);
+
+    spawnSync(`yarn add --dev chromedriver@^${majorVersion}`, {
+      stdio: 'inherit',
+      cwd: process.cwd(),
+      shell: true,
+    });
+  } else {
+    console.log(`Upgrading to chromedriver@^${majorVersion}`);
+
+    let upgraded = false;
+    const writeStream = createWriteStream('package.json.upgrading-chromedriver', { flags: 'w' });
+    const rl = createInterface({
+      input: createReadStream('package.json'),
+      crlfDelay: Infinity,
+    });
+    rl.on('line', (line) => {
+      if (line.includes('"chromedriver": "')) {
+        line = line.replace(
+          /"chromedriver":\s*"[~^]?\d[\d.]*\d"/,
+          `"chromedriver": "^${majorVersion}"`
+        );
+        upgraded = true;
+      }
+      writeStream.write(line + '\n', 'utf8');
+    });
+    rl.on('close', () => {
+      writeStream.end();
+      if (upgraded) {
+        // Remove any previous backups
+        if (existsSync('package.json.bak')) unlinkSync('package.json.bak');
+
+        renameSync('package.json', 'package.json.bak');
+        renameSync('package.json.upgrading-chromedriver', 'package.json');
+
+        console.log(`Backed up package.json and updated chromedriver to ${majorVersion}`);
+      } else {
+        unlinkSync('package.json.upgrading-chromedriver');
+        console.error(
+          `Failed to update chromedriver to ${majorVersion}. Try adding the \`--install\` switch.`
+        );
+      }
+    });
+  }
+} else {
+  console.debug(versionCheckOutput);
+  console.error(`Failed to extract the version of the installed Google Chrome.`);
+}

src/plugins/data_source/common/data_sources/types.ts

@@ -6,6 +6,7 @@
 import { SavedObjectAttributes } from 'src/core/types';
 
 export interface DataSourceAttributes extends SavedObjectAttributes {
+  id?: string;
   title: string;
   description?: string;
   endpoint: string;

src/plugins/data_source/server/client/client_pool.ts

@@ -29,7 +29,7 @@ export class OpenSearchClientPool {
 
   constructor(private logger: Logger) {}
 
-  public async setup(config: DataSourcePluginConfigType): Promise<OpenSearchClientPoolSetup> {
+  public setup(config: DataSourcePluginConfigType): OpenSearchClientPoolSetup {
     const logger = this.logger;
     const { size } = config.clientPool;
 

src/plugins/data_source/server/client/configure_client.ts

@@ -13,7 +13,7 @@ import {
 } from '../../common/data_sources';
 import { DataSourcePluginConfigType } from '../../config';
 import { CryptographyServiceSetup } from '../cryptography_service';
-import { createDataSourceError, DataSourceError } from '../lib/error';
+import { createDataSourceError } from '../lib/error';
 import { DataSourceClientParams } from '../types';
 import { parseClientOptions } from './client_config';
 import { OpenSearchClientPoolSetup } from './client_pool';
@@ -25,8 +25,8 @@ export const configureClient = async (
   logger: Logger
 ): Promise<Client> => {
   try {
-    const dataSource = await getDataSource(dataSourceId, savedObjects);
-    const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup);
+    const { attributes: dataSource } = await getDataSource(dataSourceId, savedObjects);
+    const rootClient = getRootClient(dataSource, config, openSearchClientPoolSetup);
 
     return await getQueryClient(rootClient, dataSource, cryptography);
   } catch (error: any) {
@@ -37,6 +37,43 @@ export const configureClient = async (
   }
 };
 
+export const configureTestClient = async (
+  { savedObjects, cryptography }: DataSourceClientParams,
+  dataSource: DataSourceAttributes,
+  openSearchClientPoolSetup: OpenSearchClientPoolSetup,
+  config: DataSourcePluginConfigType,
+  logger: Logger
+): Promise<Client> => {
+  try {
+    const {
+      id,
+      auth: { type, credentials },
+    } = dataSource;
+    let requireDecryption = false;
+
+    const rootClient = getRootClient(dataSource, config, openSearchClientPoolSetup);
+
+    if (type === AuthType.UsernamePasswordType && !credentials?.password && id) {
+      const { attributes: fetchedDataSource } = await getDataSource(id || '', savedObjects);
+      dataSource.auth = {
+        type,
+        credentials: {
+          username: credentials?.username || '',
+          password: fetchedDataSource.auth.credentials?.password || '',
+        },
+      };
+      requireDecryption = true;
+    }
+
+    return getQueryClient(rootClient, dataSource, cryptography, requireDecryption);
+  } catch (error: any) {
+    logger.error(`Failed to get data source client for dataSource: ${dataSource}`);
+    logger.error(error);
+    // Re-throw as DataSourceError
+    throw createDataSourceError(error);
+  }
+};
+
 export const getDataSource = async (
   dataSourceId: string,
   savedObjects: SavedObjectsClientContract
@@ -45,16 +82,17 @@ export const getDataSource = async (
     DATA_SOURCE_SAVED_OBJECT_TYPE,
     dataSourceId
   );
+
   return dataSource;
 };
 
 export const getCredential = async (
-  dataSource: SavedObject<DataSourceAttributes>,
+  dataSource: DataSourceAttributes,
   cryptography: CryptographyServiceSetup
 ): Promise<UsernamePasswordTypedContent> => {
-  const { endpoint } = dataSource.attributes!;
+  const { endpoint } = dataSource;
 
-  const { username, password } = dataSource.attributes.auth.credentials!;
+  const { username, password } = dataSource.auth.credentials!;
 
   const { decryptedText, encryptionContext } = await cryptography
     .decodeAndDecrypt(password)
@@ -87,17 +125,20 @@ export const getCredential = async (
  */
 const getQueryClient = async (
   rootClient: Client,
-  dataSource: SavedObject<DataSourceAttributes>,
-  cryptography: CryptographyServiceSetup
+  dataSource: DataSourceAttributes,
+  cryptography?: CryptographyServiceSetup,
+  requireDecryption: boolean = true
 ): Promise<Client> => {
-  const authType = dataSource.attributes.auth.type;
+  const authType = dataSource.auth.type;
 
   switch (authType) {
     case AuthType.NoAuth:
       return rootClient.child();
 
     case AuthType.UsernamePasswordType:
-      const credential = await getCredential(dataSource, cryptography);
+      const credential = requireDecryption
+        ? await getCredential(dataSource, cryptography!)
+        : (dataSource.auth.credentials as UsernamePasswordTypedContent);
       return getBasicAuthClient(rootClient, credential);
 
     default:

src/plugins/data_source/server/client/index.ts

@@ -4,4 +4,10 @@
  */
 
 export { OpenSearchClientPool, OpenSearchClientPoolSetup } from './client_pool';
-export { configureClient, getDataSource, getCredential } from './configure_client';
+export {
+  configureClient,
+  getDataSource,
+  getCredential,
+  getRootClient,
+  getValidationClient,
+} from './configure_client';