Merge branch 'main' into feature/createaggconfig-order

.github/workflows/build_and_test_workflow.yml

@@ -104,8 +104,13 @@ jobs:
           npm uninstall -g yarn
           npm i -g [email protected]
 
+      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
+      # Increase network timeout for Windows, retry once if bootstrap fails
       - name: Run bootstrap
-        run: yarn osd bootstrap
+        run: |
+          yarn cache clean
+          yarn config set network-timeout 1000000 -g
+          yarn osd bootstrap || yarn osd bootstrap
 
       - name: Run linter
         id: linter
@@ -178,6 +183,14 @@ jobs:
           JOB: ci${{ matrix.group }}
           CACHE_DIR: ciGroup${{ matrix.group }}
 
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: failure-artifacts
+          path: |
+            test/*/failure_debug/
+            test/*/screenshots/
+
   functional-tests-windows:
     runs-on: windows-latest
     name: Run functional tests on Windows
@@ -214,8 +227,13 @@ jobs:
       - name: Setup chromedriver
         run: node scripts/upgrade_chromedriver.js
 
+      # https://github.com/yarnpkg/yarn/issues/8242#issuecomment-776561223
+      # Increase network timeout for Windows, retry once if bootstrap fails
       - name: Run bootstrap
-        run: yarn osd bootstrap
+        run: |
+          yarn cache clean
+          yarn config set network-timeout 1000000 -g
+          yarn osd bootstrap || yarn osd bootstrap
 
       - name: Build plugins
         run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10
@@ -229,6 +247,14 @@ jobs:
           JOB: ci${{ matrix.group }}
           CACHE_DIR: ciGroup${{ matrix.group }}
 
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: failure-artifacts-ci${{ matrix.group }}
+          path: |
+            test/*/failure_debug/
+            test/*/screenshots/
+
   build-min-artifact-tests-linux:
     runs-on: ubuntu-latest
     container:

.node-version

@@ -1 +1 @@
-14.20.0
+14.20.1

.nvmrc

@@ -1 +1 @@
-14.20.0
+14.20.1

CHANGELOG.md

@@ -14,6 +14,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Eliminate dependency on `got` versions older than 11.8.5 ([#2801](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2801))
 - [Multi DataSource] Add explicit no spellcheck on password fields ([#2818](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2818))
 - [CVE-2022-25912] Bumps simple-git from 3.4.0 to 3.15.0 ([#3036](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3036))
+- [CVE-2022-35256] Bumps node version from 14.20.0 to 14.20.1 [#3166](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3166))
 
 ### 📈 Features/Enhancements
 
@@ -51,8 +52,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Doc] Add current plugin persistence implementation readme ([#3081](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3081))
 - [Table Visualization] Refactor table visualization using React and DataGrid component ([#2863](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2863))
 - [Vis Builder] Add redux store persistence ([#3088](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3088))
+- [Multi DataSource] Improve test connection ([#3110](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3110))
+- [Optimizer] Increase timeout waiting for the exiting of an optimizer worker ([#3193](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3193))
 - [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160))
 
+
 ### 🐛 Bug Fixes
 
 - [Vis Builder] Fixes auto bounds for timeseries bar chart visualization ([2401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2401))
@@ -81,6 +85,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Table Visualization][BUG] Fix Url content display issue in table ([#2918](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2918))
 - Fixes misleading embaddable plugin error message ([#3043](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3043))
 - [MD] Update dummy url in tests to follow lychee url allowlist ([#3099](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3099))
+- Adds config override to fix obsolete theme:version config value of v8 (beta) rendering issue ([#3045](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3045))
+- [CI] Update test workflow to increase network-timeout for yarn for installing dependencies ([#3118](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3118))
 
 ### 🚞 Infrastructure
 
@@ -89,7 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Re-enable CI workflows for feature branckes ([#2908](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2908))
 - Add Windows CI workflows ([#2966](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2966))
 - Add automatic selection of the appropriate version of chrome driver to run functional tests ([#2990](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2990))
-
+- Add recording of functional test artifacts if they fail ([#3190](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3190))
 
 ### 📝 Documentation
 
@@ -118,6 +124,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 - [Multi DataSource] Add unit test coverage for Update Data source management stack ([#2567](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2567))
 - [BWC Tests] Add BWC tests for 2.5.0 ([#2890](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2890))
+- Add retrial of flaky tests ([#2967](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2967))
 - Fix incorrect validation of time values in JUnit Reporter ([#2965](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2965))
 - Make tests covering plugin installation on cluster snapshots work across platforms ([#2994](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2994))
 - Correct the linting logic for `no-restricted-path` to ignore trailing slashes ([#3020](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3020))

DEVELOPER_GUIDE.md

@@ -36,6 +36,11 @@ $ npm i -g yarn
 $ yarn osd bootstrap # This command will also install npm dependencies
 ```
 
+If you experience a network timeout while bootstrapping, you can update the timeout by configuring it in the `.yarnrc`. For example:
+```
+network-timeout 1000000
+```
+
 ### Configure OpenSearch Dashboards
 
 *This step is only mandatory if you have https/authentication enabled, or if you use the OpenSearch Docker image in its default configuration.*

Dockerfile

@@ -1,4 +1,4 @@
-ARG NODE_VERSION=14.20.0
+ARG NODE_VERSION=14.20.1
 FROM node:${NODE_VERSION} AS base
 
 ENV HOME '.'

MAINTAINERS.md

@@ -1,13 +1,16 @@
-# OpenSearch-Dashboards Maintainers
+## Overview
+
+This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md).
+
+## Current Maintainers
 
-## Maintainers
 | Maintainer                | GitHub ID                                           | Affiliation |
 | ------------------------- | --------------------------------------------------- | ----------- |
 | Anan Zhuang               | [ananzh](https://github.com/ananzh)                 | Amazon      |
 | Bishoy Boktor             | [boktorbb](https://github.com/boktorbb)             | Amazon      |
 | Mihir Soni                | [mihirsoni](https://github.com/mihirsoni)           | Amazon      |
 | Kawika (Rocky) Avilla     | [kavilla](https://github.com/kavilla)               | Amazon      |
-| Sean Neumann              | [seanneumann](https://github.com/seanneumann)       | Amazon      | 
+| Sean Neumann              | [seanneumann](https://github.com/seanneumann)       | Amazon      |
 | Miki Barahmand            | [AMoo-Miki](https://github.com/AMoo-Miki)           | Amazon      |
 | Ashwin P Chandran         | [ashwin-pc](https://github.com/ashwin-pc)           | Amazon      |
 | Josh Romero               | [joshuarrrr](https://github.com/joshuarrrr)         | Amazon      |
@@ -19,8 +22,6 @@
 
 ## Emeritus
 
-| Maintainer                | GitHub ID                                           | Affiliation |
-| ------------------------- | --------------------------------------------------- | ----------- |
-| Tommy Markley             | [tmarkley](https://github.com/tmarkley)             | Amazon      | 
-
-[This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).
+| Maintainer    | GitHub ID                               | Affiliation |
+| ------------- | --------------------------------------- | ----------- |
+| Tommy Markley | [tmarkley](https://github.com/tmarkley) | Amazon      |

TESTING.md

@@ -24,7 +24,7 @@ In general, we recommend four tiers of tests:
 
 # Requirements
 * Install the latest NodeJS, [NPM](https://www.npmjs.com/get-npm) and [Yarn](https://classic.yarnpkg.com/en/docs/install/#mac-stable)
-    * `nvm install v14.20.0`
+    * `nvm install v14.20.1`
     * `npm install -g yarn`
 
 # Running tests

package.json

@@ -94,7 +94,9 @@
     "**/qs": "^6.10.3",
     "**/trim": "^0.0.3",
     "**/typescript": "4.0.2",
-    "**/unset-value": "^2.0.1"
+    "**/unset-value": "^2.0.1",
+    "**/jest-config": "npm:@amoo-miki/[email protected]",
+    "**/jest-jasmine2": "npm:@amoo-miki/[email protected]"
   },
   "workspaces": {
     "packages": [
@@ -463,7 +465,7 @@
     "zlib": "^1.0.5"
   },
   "engines": {
-    "node": "14.20.0",
+    "node": "14.20.1",
     "yarn": "^1.21.1"
   }
 }

packages/osd-optimizer/src/worker/run_worker.ts

@@ -78,11 +78,13 @@ const exit = (code: number) => {
   setTimeout(() => {
     send(
       workerMsgs.error(
-        new Error('process did not automatically exit within 5 seconds, forcing exit')
+        new Error(
+          `process did not automatically exit within 15 seconds (previous code: ${code}); forcing exit...`
+        )
       )
     );
     process.exit(1);
-  }, 5000).unref();
+  }, 15000).unref();
 };
 
 // check for connected parent on an unref'd timer rather than listening

src/core/server/ui_settings/ui_settings_client.test.ts

@@ -624,6 +624,29 @@ describe('ui settings', () => {
       expect(await uiSettings.get('dateFormat')).toBe('foo');
     });
 
+    it('returns the overridden value for key theme:version', async () => {
+      const opensearchDocSource = { 'theme:version': 'v8 (beta)' };
+      const overrides = { 'theme:version': 'v7' };
+      const { uiSettings } = setup({ opensearchDocSource, overrides });
+
+      expect(await uiSettings.get('theme:version')).toBe('v7');
+    });
+
+    it('returns the overridden value for key theme:version when doc source is empty', async () => {
+      const opensearchDocSource = {};
+      const overrides = { 'theme:version': 'v7' };
+      const { uiSettings } = setup({ opensearchDocSource, overrides });
+
+      expect(await uiSettings.get('theme:version')).toBe('v7');
+    });
+
+    it('rewrites the key theme:version value without override', async () => {
+      const opensearchDocSource = { 'theme:version': 'v8 (beta)' };
+      const { uiSettings } = setup({ opensearchDocSource });
+
+      expect(await uiSettings.get('theme:version')).toBe('v8 (beta)');
+    });
+
     it('returns the default value for an override with value null', async () => {
       const opensearchDocSource = { dateFormat: 'YYYY-MM-DD' };
       const overrides = { dateFormat: null };

src/core/server/ui_settings/ui_settings_config.ts

@@ -38,7 +38,12 @@ const deprecations: ConfigDeprecationProvider = ({ unused, renameFromRoot }) =>
 ];
 
 const configSchema = schema.object({
-  overrides: schema.object({}, { unknowns: 'allow' }),
+  overrides: schema.object(
+    {
+      'theme:version': schema.string({ defaultValue: 'v7' }),
+    },
+    { unknowns: 'allow' }
+  ),
 });
 
 export type UiSettingsConfigType = TypeOf<typeof configSchema>;

src/dev/jest/config.js

@@ -109,4 +109,5 @@ export default {
   globals: {
     Uint8Array: Uint8Array,
   },
+  flakyTestRetries: 2,
 };

src/plugins/data_source/common/data_sources/types.ts

@@ -6,7 +6,6 @@
 import { SavedObjectAttributes } from 'src/core/types';
 
 export interface DataSourceAttributes extends SavedObjectAttributes {
-  id?: string;
   title: string;
   description?: string;
   endpoint: string;

src/plugins/data_source/server/client/configure_client.ts

@@ -25,7 +25,7 @@ export const configureClient = async (
   logger: Logger
 ): Promise<Client> => {
   try {
-    const { attributes: dataSource } = await getDataSource(dataSourceId, savedObjects);
+    const { attributes: dataSource } = await getDataSource(dataSourceId!, savedObjects);
     const rootClient = getRootClient(dataSource, config, openSearchClientPoolSetup);
 
     return await getQueryClient(rootClient, dataSource, cryptography);
@@ -38,36 +38,29 @@ export const configureClient = async (
 };
 
 export const configureTestClient = async (
-  { savedObjects, cryptography }: DataSourceClientParams,
+  { savedObjects, cryptography, dataSourceId }: DataSourceClientParams,
   dataSource: DataSourceAttributes,
   openSearchClientPoolSetup: OpenSearchClientPoolSetup,
   config: DataSourcePluginConfigType,
   logger: Logger
 ): Promise<Client> => {
   try {
     const {
-      id,
       auth: { type, credentials },
     } = dataSource;
     let requireDecryption = false;
 
     const rootClient = getRootClient(dataSource, config, openSearchClientPoolSetup);
 
-    if (type === AuthType.UsernamePasswordType && !credentials?.password && id) {
-      const { attributes: fetchedDataSource } = await getDataSource(id || '', savedObjects);
-      dataSource.auth = {
-        type,
-        credentials: {
-          username: credentials?.username || '',
-          password: fetchedDataSource.auth.credentials?.password || '',
-        },
-      };
+    if (type === AuthType.UsernamePasswordType && !credentials?.password && dataSourceId) {
+      const dataSourceSavedObject = await getDataSource(dataSourceId, savedObjects);
+      dataSource = dataSourceSavedObject.attributes;
       requireDecryption = true;
     }
 
     return getQueryClient(rootClient, dataSource, cryptography, requireDecryption);
   } catch (error: any) {
-    logger.error(`Failed to get data source client for dataSource: ${dataSource}`);
+    logger.error(`Failed to get test client for dataSource: ${dataSource}`);
     logger.error(error);
     // Re-throw as DataSourceError
     throw createDataSourceError(error);

src/plugins/data_source/server/data_source_service.ts

@@ -40,8 +40,8 @@ export class DataSourceService {
   }
 
   async setup(config: DataSourcePluginConfigType): Promise<DataSourceServiceSetup> {
-    const opensearchClientPoolSetup = await this.openSearchClientPool.setup(config);
-    const legacyClientPoolSetup = await this.legacyClientPool.setup(config);
+    const opensearchClientPoolSetup = this.openSearchClientPool.setup(config);
+    const legacyClientPoolSetup = this.legacyClientPool.setup(config);
 
     const getDataSourceClient = async (
       params: DataSourceClientParams

src/plugins/data_source/server/legacy/configure_legacy_client.test.ts

@@ -19,7 +19,7 @@ import { configureLegacyClient } from './configure_legacy_client';
 const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8';
 
 // TODO: improve UT
-describe.skip('configureLegacyClient', () => {
+describe('configureLegacyClient', () => {
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
   let config: DataSourcePluginConfigType;
   let savedObjectsMock: jest.Mocked<SavedObjectsClientContract>;

src/plugins/data_source/server/legacy/configure_legacy_client.ts

@@ -33,10 +33,10 @@ export const configureLegacyClient = async (
   logger: Logger
 ) => {
   try {
-    const dataSource = await getDataSource(dataSourceId, savedObjects);
+    const dataSource = await getDataSource(dataSourceId!, savedObjects);
     const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup);
 
-    return await getQueryClient(rootClient, dataSource, cryptography, callApiParams);
+    return await getQueryClient(rootClient, dataSource.attributes, cryptography, callApiParams);
   } catch (error: any) {
     logger.error(`Failed to get data source client for dataSourceId: [${dataSourceId}]`);
     logger.error(error);
@@ -55,11 +55,11 @@ export const configureLegacyClient = async (
  */
 const getQueryClient = async (
   rootClient: Client,
-  dataSource: SavedObject<DataSourceAttributes>,
+  dataSource: DataSourceAttributes,
   cryptography: CryptographyServiceSetup,
   { endpoint, clientParams, options }: LegacyClientCallAPIParams
 ) => {
-  const authType = dataSource.attributes.auth.type;
+  const authType = dataSource.auth.type;
 
   switch (authType) {
     case AuthType.NoAuth:

src/plugins/data_source/server/lib/error.ts

@@ -11,9 +11,18 @@ import { OsdError } from '../../../opensearch_dashboards_utils/common';
 export class DataSourceError extends OsdError {
   // must have statusCode to avoid route handler in search.ts to return 500
   statusCode: number;
-  constructor(error: any, message?: string, statusCode?: number) {
-    message = message ? message : error.message;
+  constructor(error: any, context?: string, statusCode?: number) {
+    let message: string;
+    if (context) {
+      message = context;
+    } else if (isResponseError(error)) {
+      message = JSON.stringify(error.meta.body);
+    } else {
+      message = error.message;
+    }
+
     super('Data Source Error: ' + message);
+
     if (statusCode) {
       this.statusCode = statusCode;
     } else if (error.statusCode) {

src/plugins/data_source/server/routes/data_source_connection_validator.ts

@@ -13,11 +13,7 @@ export class DataSourceConnectionValidator {
     try {
       return await this.callDataCluster.info<OpenSearchClient>();
     } catch (e) {
-      if (e.statusCode === 403) {
-        return true;
-      } else {
-        throw createDataSourceError(e);
-      }
+      throw createDataSourceError(e);
     }
   }
 }