S3 dataset tests env

.github/workflows/cypress_workflow.yml

@@ -41,6 +41,9 @@ env:
   COMMENT_SUCCESS_MSG: ':white_check_mark: Cypress test run succeeded!'
   COMMENT_FAILURE_MSG: ':x: Cypress test run failed!'
   LATEST_VERSION: '2.17.0'
+  S3_CONNECTION_URL: ${{ secrets.S3_CONNECTION_URL }}
+  S3_CONNECTION_USERNAME: ${{ secrets.S3_CONNECTION_USERNAME }}
+  S3_CONNECTION_PASSWORD: ${{ secrets.S3_CONNECTION_PASSWORD }}
 
 jobs:
   cypress-tests:

.github/workflows/s3_dataset_cypress_workflow.yml

@@ -0,0 +1,148 @@
+name: Run cypress tests (CI Group 10)
+
+# trigger once approved
+on:
+  pull_request:
+    branches: ['**']
+    paths-ignore:
+      - '**/*.md'
+      - '.lycheeignore'
+      - 'changelogs/fragments/**'
+  workflow_dispatch:
+    inputs:
+      test_repo:
+        description: 'Cypress test repo'
+        default: 'opensearch-project/opensearch-dashboards-functional-test'
+        required: true
+        type: string
+      test_branch:
+        description: 'Cypress test branch (default: source branch)'
+        required: false
+        type: string
+      specs:
+        description: 'Tests to run (default: osd:ciGroup10)'
+        required: false
+        type: string
+      pr_number:
+        description: 'PR Number (optional)'
+        required: false
+        type: number
+
+env:
+  TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || 'opensearch-project/opensearch-dashboards-functional-test' }}
+  TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}"
+  FTR_PATH: 'ftr'
+  CYPRESS_BROWSER: 'chromium'
+  CYPRESS_VISBUILDER_ENABLED: true
+  CYPRESS_DATASOURCE_MANAGEMENT_ENABLED: false
+  OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true
+  NODE_OPTIONS: '--max-old-space-size=6144 --dns-result-order=ipv4first'
+  LATEST_VERSION: '2.17.0'
+
+jobs:
+  cypress-tests:
+    runs-on: ubuntu-latest
+    environment: flint-test
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
+    env:
+      START_CMD: 'node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=true --uiSettings.overrides[''home:useNewHomePage'']=true --data_source.enabled=true --workspace.enabled=true --opensearch.ignoreVersionMismatch=true'
+      OPENSEARCH_SNAPSHOT_CMD: '/bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"'
+    name: Run cypress tests (osd:ciGroup10)
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version-file: '.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g [email protected]
+
+      - name: Run bootstrap
+        run: yarn osd bootstrap
+
+      - name: Build plugins
+        run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12
+
+      - name: Checkout FT repo
+        uses: actions/checkout@v2
+        with:
+          path: ${{ env.FTR_PATH }}
+          repository: ${{ env.TEST_REPO }}
+          ref: '${{ env.TEST_BRANCH }}'
+
+      - name: Setup spec files
+        if: ${{ inputs.specs == '' }}
+        shell: bash
+        run: |
+          DASHBOARDS_SPEC="$(yarn --silent osd:ciGroup10)"
+          echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV
+          echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}"
+
+      - name: Download OpenSearch
+        uses: suisei-cn/[email protected]
+        with:
+          url: https://artifacts.opensearch.org/releases/bundle/opensearch/${{ env.LATEST_VERSION }}/opensearch-${{ env.LATEST_VERSION }}-linux-x64.tar.gz
+
+      - name: Extract OpenSearch
+        run: |
+          tar -xzf opensearch-*.tar.gz
+          rm -f opensearch-*.tar.gz
+        shell: bash
+
+      - name: Remove security plugin
+        run: |
+          /bin/bash -c "yes | ./opensearch-${{ env.LATEST_VERSION }}/bin/opensearch-plugin remove opensearch-security"
+        shell: bash
+
+      - name: Run OpenSearch
+        run: |
+          /bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"
+          sleep 30
+        shell: bash
+
+      - name: Clear Cypress Cache
+        run: npx cypress cache clear
+
+      - name: Run Dashboards Cypress tests with query enhancements
+        uses: cypress-io/github-action@v6
+        env:
+          S3_CONNECTION_URL: ${{ secrets.S3_CONNECTION_URL }}
+          S3_CONNECTION_USERNAME: ${{ secrets.S3_CONNECTION_USERNAME }}
+          S3_CONNECTION_PASSWORD: ${{ secrets.S3_CONNECTION_PASSWORD }}
+        with:
+          install-command: npx cypress install --force
+          start: ${{ env.START_CMD }}
+          wait-on: 'http://localhost:9200, http://localhost:5601'
+          command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
+
+      - name: Upload Dashboards screenshots
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: dashboards-cypress-screenshots-10
+          path: cypress/screenshots
+          retention-days: 1
+
+      - name: Upload Dashboards repo videos
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: dashboards-cypress-videos-10
+          path: cypress/videos
+          retention-days: 1
+
+      - name: Upload Dashboards repo results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: dashboards-cypress-results-10
+          path: cypress/results
+          retention-days: 1

cypress.config.ts

@@ -21,6 +21,12 @@ module.exports = defineConfig({
       name: 'test_cluster',
       url: 'http://localhost:9200',
     },
+    S3_ENGINE: {
+      name: 'BasicS3Connection',
+      url: process.env.S3_CONNECTION_URL,
+      username: process.env.S3_CONNECTION_USERNAME,
+      password: process.env.S3_CONNECTION_PASSWORD,
+    },
     openSearchUrl: 'http://localhost:9200',
     SECURITY_ENABLED: false,
     AGGREGATION_VIEW: false,

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/query_enhancements/s3_dataset.spec.js

@@ -0,0 +1,137 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  DS_API,
+  DSM_API,
+  S3_CLUSTER,
+} from '../../../../../utils/apps/query_enhancements/constants';
+import { WORKSPACE_NAME } from './constants';
+
+let dataSourceId = '';
+
+describe('S3 Dataset', { defaultCommandTimeout: 120000 }, () => {
+  before(() => {
+    cy.request({
+      method: 'POST',
+      url: `${DSM_API}`,
+      headers: {
+        'osd-xsrf': true,
+      },
+      body: {
+        dataSourceAttr: {
+          endpoint: S3_CLUSTER.url,
+          auth: {
+            type: 'username_password',
+            credentials: {
+              username: S3_CLUSTER.username,
+              password: S3_CLUSTER.password,
+            },
+          },
+        },
+      },
+    }).then((metaRes) => {
+      if (metaRes && metaRes.body) {
+        cy.request({
+          method: 'POST',
+          url: `${DS_API.CREATE_DATA_SOURCE}`,
+          headers: {
+            'osd-xsrf': true,
+            'Content-Type': 'application/json',
+          },
+          body: {
+            attributes: {
+              title: S3_CLUSTER.name,
+              description: '',
+              ...metaRes.body,
+              endpoint: S3_CLUSTER.url,
+              auth: {
+                type: 'username_password',
+                credentials: {
+                  username: S3_CLUSTER.username,
+                  password: S3_CLUSTER.password,
+                },
+              },
+            },
+          },
+        }).then((resp) => {
+          if (resp && resp.body && resp.body.id) {
+            dataSourceId = resp.body.id;
+          }
+        });
+      }
+    });
+  });
+
+  after(() => {
+    cy.request({
+      method: 'DELETE',
+      url: `${DS_API.DELETE_DATA_SOURCE}${dataSourceId}`,
+      body: { force: false },
+      headers: {
+        'osd-xsrf': true,
+      },
+    });
+  });
+
+  describe('Run S3 Query', () => {
+    beforeEach(() => {
+      // Create workspace
+      cy.deleteWorkspaceByName(WORKSPACE_NAME);
+      cy.visit('/app/home');
+      cy.createInitialWorkspaceWithDataSource(S3_CLUSTER.name, WORKSPACE_NAME);
+    });
+    afterEach(() => {
+      cy.deleteWorkspaceByName(WORKSPACE_NAME);
+    });
+
+    it('with SQL', function () {
+      cy.getElementByTestId(`datasetSelectorButton`).click();
+      cy.getElementByTestId(`datasetSelectorAdvancedButton`).click();
+
+      cy.get(`[title="S3 Connections"]`).click();
+      cy.get(`[title="BasicS3Connection"]`).click();
+      cy.get(`[title="mys3"]`).click();
+      cy.get(`[title="default"]`).click();
+      cy.get(`[title="http_logs"]`).click();
+      cy.getElementByTestId('datasetSelectorNext').click();
+      cy.get(`[class="euiModalHeader__title"]`).should('contain', 'Step 2: Configure data');
+
+      cy.getElementByTestId('advancedSelectorLanguageSelect').select('OpenSearch SQL');
+      cy.getElementByTestId('advancedSelectorConfirmButton').click();
+      cy.waitForLoader(true);
+      cy.waitForSearch();
+
+      cy.getElementByTestId('queryEditorLanguageSelector').should('contain', 'OpenSearch SQL');
+      cy.get(`[data-test-subj="queryResultCompleteMsg"]`).should('be.visible');
+      cy.getElementByTestId('docTable').should('be.visible');
+      cy.getElementByTestId('docTable').find('tr').should('have.length', 11);
+    });
+
+    // Skipping until #8922 is merged in
+    it.skip('with PPL', function () {
+      cy.getElementByTestId(`datasetSelectorButton`).click();
+      cy.getElementByTestId(`datasetSelectorAdvancedButton`).click();
+
+      cy.get(`[title="S3 Connections"]`).click();
+      cy.get(`[title="BasicS3Connection"]`).click();
+      cy.get(`[title="mys3"]`).click();
+      cy.get(`[title="default"]`).click();
+      cy.get(`[title="http_logs"]`).click();
+      cy.getElementByTestId('datasetSelectorNext').click();
+      cy.get(`[class="euiModalHeader__title"]`).should('contain', 'Step 2: Configure data');
+
+      cy.getElementByTestId('advancedSelectorLanguageSelect').select('PPL');
+      cy.getElementByTestId('advancedSelectorConfirmButton').click();
+      cy.waitForLoader(true);
+      cy.waitForSearch();
+
+      cy.getElementByTestId('queryEditorLanguageSelector').should('contain', 'PPL');
+      cy.get(`[data-test-subj="queryResultCompleteMsg"]`).should('be.visible');
+      cy.getElementByTestId('docTable').should('be.visible');
+      cy.getElementByTestId('docTable').find('tr').should('have.length', 11);
+    });
+  });
+});

cypress/utils/apps/query_enhancements/constants.js

@@ -5,3 +5,13 @@
 
 export const clusterName = 'test_cluster';
 export const clusterConnection = 'http://localhost:9200';
+
+export const S3_CLUSTER = Cypress.env('S3_ENGINE');
+
+export const DS_API_PREFIX = '/api/saved_objects';
+export const DS_API = {
+  DATA_SOURCES_LISTING: `${DS_API_PREFIX}/_find?fields=id&fields=description&fields=title&per_page=10000&type=data-source`,
+  CREATE_DATA_SOURCE: `${DS_API_PREFIX}/data-source`,
+  DELETE_DATA_SOURCE: `${DS_API_PREFIX}/data-source/`,
+};
+export const DSM_API = '/internal/data-source-management/fetchDataSourceMetaData';