azure-pipelines-expectation-gallery.yml

# This file is responsible for configuring the `expectation_gallery` pipeline (https://dev.azure.com/great-expectations/great_expectations/_build)
#
# The pipeline is run under the following conditions:
#   - On the develop branch when a weekly release is being cut
#   - On the develop branch as scheduled by the below cron job
#
# The sole purpose of this pipeline is to build and publish the Expectation gallery. As such, it is designed to run quickly and frequently
# to ensure that the gallery is kept to-up-date with change.

schedules:
- cron: 0 */2 * * *
  displayName: Scheduled Runs
  branches:
    include:
    - develop
  always: false # Will only trigger if the state of the codebase has changed sinced the last scheduled run

resources:
  containers:
  - container: postgres
    image: postgres:11
    ports:
    - 5432:5432
    env:
      POSTGRES_DB: "test_ci"
      POSTGRES_HOST_AUTH_METHOD: "trust"
  - container: mysql
    image: mysql:8.0.20
    ports:
      - 3306:3306
    env:
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: test_ci
  - container: mssql
    image: mcr.microsoft.com/mssql/server:2019-latest
    env:
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: ReallyStrongPwd1234%^&*
      MSSQL_DB: test_ci
      MSSQL_PID: Developer
    ports:
      - 1433:1433
  - container: trino
    image: trinodb/trino:379
    ports:
      - 8088:8080

# The pipeline is run under two primary conditions: if cutting a release or as scheduled by the above cron job.
variables:
  isRelease: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')]
  isScheduled: $[and(eq(variables['Build.SourceBranch'], 'refs/heads/develop'), eq(variables['Build.Reason'], 'Schedule'))]
  isManual: $[eq(variables['Build.Reason'], 'Manual')]

stages:
  - stage: scope_check
    pool:
      vmImage: 'ubuntu-20.04'
    jobs:
      - job: changes
        steps:
          - task: ChangedFiles@1
            name: CheckChanges
            inputs:
              verbose: true
              rules: |
                [ContribChanged]
                contrib/**

                [GEChanged]
                great_expectations/**
                tests/**

  - stage: deploy_gallery
    dependsOn: scope_check
    condition: or(eq(variables.isScheduled, true), eq(variables.isRelease, true), eq(variables.isManual, true))
    pool:
      vmImage: 'ubuntu-18.04'

    jobs:
      - job: build_gallery
        variables:
          python.version: '3.7'
          GE_pytest_pip_opts: '"pyspark<3.0.0" "cookiecutter==1.7.3" "google-cloud-bigquery-storage" --requirement requirements-dev.txt --constraint constraints-dev.txt'
          GE_pytest_opts: ''

        services:
          postgres: postgres
          mysql: mysql
          mssql: mssql
          trino: trino

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip==21.3.1
            displayName: 'Update pip'

          - script: |
              pip install $(GE_pytest_pip_opts)
              pip install --requirement requirements.txt
              pip install  --editable .
            displayName: 'Install dependencies'

          - script: |
              printf 'Waiting for MySQL database to accept connections'
              until mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --execute "SHOW DATABASES"; do
                printf '.'
                sleep 1;
              done;
            displayName: 'Wait for MySQL database to initialise'

          - script: |
              echo "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" > mysql_setup_script.sql
              mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --reconnect < mysql_setup_script.sql
            displayName: 'Configure mysql'

          - script: |
              sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt
            displayName: 'Configure mssql'

          - script: |
              printf 'Waiting for Trino database to accept connections'
              sleep 30
#             until trino --execute "SHOW CATALOGS"; do
#               printf '.'
#               sleep 1;
#             done;
            displayName: 'Wait for Trino database to initialise'

#         - task: DownloadSecureFile@1
#           name: gcp_authkey
#           displayName: 'Download Google Service Account'
#           inputs:
#             secureFile: 'superconductive-service-acct_ge-oss-ci-cd.json'
#             retryCount: '2'

          - bash: python ./build_gallery.py
            workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
            displayName: 'Build Gallery'
#           env:
#             GOOGLE_APPLICATION_CREDENTIALS: $(gcp_authkey.secureFilePath)
#             GE_TEST_GCP_PROJECT: $(GE_TEST_GCP_PROJECT)
#             GE_TEST_BIGQUERY_DATASET: $(GE_TEST_BIGQUERY_DATASET)
#             GE_TEST_BIGQUERY_PERFORMANCE_DATASET: $(GE_TEST_BIGQUERY_PERFORMANCE_DATASET)

          - bash: python ./build_package_gallery.py
            workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
            displayName: 'Build Package Gallery'

          - task: S3Upload@1
            inputs:
              regionName: 'us-east-2'
              awsCredentials: 'aws-ci-great-expectations'
              bucketName: 'superconductive-public'
              sourceFolder: '$(Build.SourcesDirectory)/assets/scripts'
              globExpressions: '*.json'
              targetFolder: 'static/gallery/'
              filesAcl: 'public-read'

          - bash: |
              echo "About to trigger webhook: $GALLERY_BUILD_HOOK"
              curl -X POST -d {} $GALLERY_BUILD_HOOK
            displayName: 'Trigger gallery build'
            env:
              GALLERY_BUILD_HOOK: $(gallerywebhook)