Fix flaky test for vis-augmenter

.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml

@@ -17,14 +17,15 @@ jobs:
           filters: |
             tests:
               - 'cypress/**/core-opensearch-dashboards/**'
+              - 'cypress/utils/dashboards/**'
 
   tests-with-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
 
@@ -34,7 +35,7 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
       security-enabled: false

.github/workflows/cypress-workflow-bundle-snapshot-based.yml

@@ -17,14 +17,15 @@ jobs:
           filters: |
             tests:
               - 'cypress/**/core-opensearch-dashboards/**'
+              - 'cypress/utils/dashboards/**'
 
   tests-with-security:
     needs: changes
     if: ${{ needs.changes.outputs.tests == 'true' }}
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
 
   tests-without-security:
@@ -33,6 +34,6 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true --ml_commons_dashboards.enabled=true
       security-enabled: false

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js

@@ -12,6 +12,7 @@ import {
   deleteVisAugmenterData,
   bootstrapDashboard,
 } from '../../../../../utils/dashboards/vis-augmenter/helpers';
+import { CURRENT_TENANT } from '../../../../../utils/commands';
 
 describe('Vis augmenter - existing dashboards work as expected', () => {
   describe('dashboard with ineligible, eligible, and vega visualizations', () => {
@@ -79,6 +80,7 @@ describe('Vis augmenter - existing dashboards work as expected', () => {
     );
 
     before(() => {
+      CURRENT_TENANT.newTenant = 'global';
       // Create a dashboard and add some visualizations
       bootstrapDashboard(
         INDEX_SETTINGS_FILEPATH_SIMPLE,

cypress/utils/dashboards/vis-augmenter/commands.js

@@ -32,8 +32,9 @@ Cypress.Commands.add('getMenuItems', { prevSubject: 'optional' }, (menu) =>
 );
 
 Cypress.Commands.add('visitDashboard', (dashboardName) => {
+  cy.intercept('/api/saved_objects/_find*').as('loadDashboards');
   cy.visit(`${BASE_PATH}/app/dashboards`);
-  cy.wait(2000);
+  cy.wait('@loadDashboards', { timeout: 300000 });
   cy.get('.euiFieldSearch').type(dashboardName);
   cy.wait(2000);
   cy.get('[data-test-subj="itemsInMemTable"]').contains(dashboardName).click({

cypress/utils/dashboards/vis-augmenter/helpers.js

@@ -178,7 +178,6 @@ const addMetric = (metric, index) => {
           force: true,
         }
       );
-      cy.contains(`${metric.aggregation}`).click({ force: true });
     });
 
   // non-count aggregations will have an additional field value to set
@@ -257,7 +256,9 @@ export const bootstrapDashboard = (
     sampleDataFilepath
   );
 
+  cy.intercept('/api/saved_objects/*').as('savedObjectsApis');
   miscUtils.visitPage('app/dashboards#/create');
+  cy.wait(['@savedObjectsApis'], { timeout: 300000 });
 
   // Create several different visualizations
   visualizationSpecs.forEach((visualizationSpec) => {
@@ -267,9 +268,35 @@ export const bootstrapDashboard = (
       visualizationSpec.name,
       visualizationSpec.metrics
     );
+    // wait for vis saved
+    cy.contains(`Saved '${visualizationSpec.name}'`);
+    cy.wait(5000);
+    // cy.get(`[data-title="${visualizationSpec.name}"]`).should('have.length', 1);
+    // somehow the visualization is not added to dashboard
+    cy.get('body').then(($body) => {
+      if ($body.find(`[data-title="${visualizationSpec.name}"]`).length === 0) {
+        cy.getElementByTestId('dashboardAddPanelButton').click({ force: true });
+        cy.getElementByTestId('savedObjectFinderSearchInput')
+          .focus()
+          .clear()
+          .type(visualizationSpec.name);
+
+        cy.getElementByTestId(
+          `savedObjectTitle${visualizationSpec.name}`
+        ).click();
+
+        cy.getElementByTestId('euiFlyoutCloseButton').click();
+
+        cy.get(`[data-title="${visualizationSpec.name}"]`).should(
+          'have.length',
+          1
+        );
+      }
+    });
   });
 
-  cy.getElementByTestId('dashboardSaveMenuItem').click({
+  /// wait for page load
+  cy.getElementByTestId('dashboardSaveMenuItem', { timeout: 300000 }).click({
     force: true,
   });
 
@@ -278,6 +305,11 @@ export const bootstrapDashboard = (
   cy.getElementByTestId('confirmSaveSavedObjectButton').click({
     force: true,
   });
+
+  cy.contains(`Dashboard '${dashboardName}' was saved`);
+
+  // make newly created dashboards searchable
+  devToolsRequest('.kibana*/_refresh', 'POST');
 };
 
 export const filterByObjectType = (type) => {