Add a documentation page describing the different parts of the Openverse stack #7744
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI + CD | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: "The tag to assign to the images built in the workflow." | |
| type: string | |
| required: false | |
| default: "" | |
| # This is useful when dispatching on a branch other than `main`. | |
| perform_deploy: | |
| description: "Publish images and deploy staging?" | |
| type: boolean | |
| required: false | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| # Don't continue building images for a PR if the PR is updated quickly | |
| # For other workflows, allow them to complete and just block on them. This | |
| # ensures deployments in particular to happen in series rather than parallel. | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| ########### | |
| # Helpers # | |
| ########### | |
| get-changes: | |
| name: Get changes | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| changes: ${{ steps.paths-filter.outputs.changes }} | |
| catalog: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'catalog') }} | |
| ingestion_server: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ingestion_server') }} | |
| api: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'api') }} | |
| frontend: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'frontend') }} | |
| documentation: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'documentation') }} | |
| ci_cd: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ci_cd') }} | |
| packages: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'packages') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get changes | |
| id: paths-filter | |
| uses: ./.github/actions/get-changes | |
| get-image-tag: | |
| name: Get image tag | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.get-image-tag.outputs.image_tag }} | |
| steps: | |
| - name: Get image tag | |
| id: get-image-tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.image_tag }}" != "" ]]; then | |
| echo "image_tag=${{ inputs.image_tag }}" | tee "$GITHUB_OUTPUT" | |
| else | |
| echo "image_tag=${{ github.sha }}" | tee "$GITHUB_OUTPUT" | |
| fi | |
| determine-images: | |
| name: Determine images to build and publish | |
| runs-on: ubuntu-latest | |
| outputs: | |
| do_build: ${{ steps.set-matrix.outputs.do_build }} | |
| build_matrix: ${{ steps.set-matrix.outputs.build_matrix }} | |
| do_publish: ${{ steps.set-matrix.outputs.do_publish }} | |
| publish_matrix: ${{ steps.set-matrix.outputs.publish_matrix }} | |
| needs: | |
| - get-changes | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set matrix images | |
| id: set-matrix | |
| env: | |
| CHANGES: ${{ needs.get-changes.outputs.changes }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python set_matrix_images.py | |
| ############# | |
| # Universal # | |
| ############# | |
| lint: # This includes type-checking of the frontend. | |
| name: Lint files | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| # No Python setup is needed, the built-in Python runtime is okay. | |
| setup_python: false | |
| # Node.js is needed by lint actions. | |
| install_recipe: "node-install" | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Run pre-commit to lint files | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| just precommit | |
| just lint | |
| validate-codeowners: | |
| name: Validate CODEOWNERS | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: mszostok/[email protected] | |
| with: | |
| checks: "files,duppatterns,syntax" | |
| experimental_checks: "notowned,avoid-shadowing" | |
| add-stack-label: | |
| name: Add stack label | |
| if: | | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.owner.login == 'WordPress' && | |
| github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| steps: | |
| - name: Apply stack labels | |
| uses: actions/github-script@v6 | |
| with: | |
| # Also see | |
| # - list of stack filters: `.github/filters.yml` | |
| # - list of stack labels: https://github.com/WordPress/openverse/labels?q=stack | |
| script: | | |
| const labels = await github.rest.issues.listLabelsOnIssue({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| }) | |
| console.log(labels.data) | |
| if (labels.data.some(label => label.name.startsWith("🧱 stack: "))) { | |
| console.log("Stack label already applied, skipping.") | |
| } else { | |
| const stacks = ["catalog", "api", "ingestion_server", "frontend", "documentation", "mgmt"] | |
| const labels = JSON | |
| .parse('${{ needs.get-changes.outputs.changes }}') | |
| .filter(change => stacks.includes(change)) | |
| .map(change => `🧱 stack: ${change.replace("_", " ")}`) | |
| if (!labels.length) { | |
| console.log("Couldn't determine stack, applying triage labels.") | |
| labels.push("🚦 status: awaiting triage", "🏷 status: label work required") | |
| } | |
| await github.rest.issues.addLabels({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels, | |
| }) | |
| } | |
| build-images: | |
| name: Build Docker images | |
| if: needs.determine-images.outputs.do_build == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-images.outputs.build_matrix) }} | |
| needs: | |
| - get-image-tag | |
| - lint | |
| - determine-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # Sets up `just` and Node.js. | |
| # Node.js is needed for the frontend to download translations. | |
| # `just` is needed to run the recipes and set build args. | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| setup_nodejs: ${{ matrix.image == 'frontend' && 'true' || 'false' }} | |
| install_recipe: ${{ matrix.image == 'frontend' && 'node-install' || '' }} | |
| # Sets build args specifying versions needed to build Docker image. | |
| - name: Prepare build args | |
| id: prepare-build-args | |
| run: | | |
| just versions | tee "$GITHUB_OUTPUT" | |
| # ℹ️Step only applies for frontend image. | |
| # This step | |
| # - downloads translation strings from GlotPress so that they can be | |
| # bundled inside the Docker image | |
| - name: Prepare frontend for building | |
| if: matrix.image == 'frontend' | |
| run: | | |
| just frontend/run i18n | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| install: true | |
| - name: Build image `${{ matrix.image }}` | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ matrix.context }} | |
| target: ${{ matrix.target || '' }} | |
| push: false | |
| tags: openverse-${{ matrix.image }} | |
| file: ${{ matrix.file }} | |
| cache-from: type=gha,scope=${{ matrix.image }} | |
| cache-to: type=gha,scope=${{ matrix.image }} | |
| outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar | |
| build-contexts: ${{ matrix.build-contexts || '' }} | |
| build-args: | | |
| SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} | |
| CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }} | |
| CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }} | |
| API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }} | |
| INGESTION_PY_VERSION=${{ steps.prepare-build-args.outputs.ingestion_py_version }} | |
| FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }} | |
| FRONTEND_PNPM_VERSION=${{ steps.prepare-build-args.outputs.frontend_pnpm_version }} | |
| PGCLI_VERSION=${{ steps.prepare-build-args.outputs.pgcli_version }} | |
| - name: Upload image `${{ matrix.image }}` | |
| id: upload-img | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.image }} | |
| path: /tmp/${{ matrix.image }}.tar | |
| - name: Show artifact ID | |
| run: | | |
| echo '${{ matrix.image }} artifact ID is ${{ steps.upload-img.outputs.artifact-id }}' | |
| ########### | |
| # Catalog # | |
| ########### | |
| test-cat: | |
| name: Run tests for the catalog | |
| if: | | |
| needs.get-changes.outputs.catalog == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run the tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db catalog | |
| - name: Run tests | |
| run: | | |
| just catalog/test --extended | |
| catalog-checks: | |
| name: Run catalog checks | |
| if: | | |
| needs.get-changes.outputs.catalog == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| # Python is not needed to run the tests. | |
| setup_python: false | |
| # Node.js is needed by lint actions. | |
| install_recipe: "node-install" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db catalog | |
| - name: Check DAG documentation | |
| run: | | |
| just catalog/generate-dag-docs true | |
| #################### | |
| # Ingestion server # | |
| #################### | |
| test-ing: | |
| name: Run tests for ingestion-server | |
| if: | | |
| needs.get-changes.outputs.ingestion_server == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| # Python is needed to run the test. | |
| setup_nodejs: false | |
| # Node.js is not needed to run ingestion server tests. | |
| install_recipe: ingestion_server/install | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server | |
| - name: Run ingestion-server tests | |
| run: just ingestion_server/test-local | |
| - name: Print ingestion-server test logs | |
| run: | | |
| just ingestion_server/test-logs > ingestion_server/test/ingestion_logs.txt | |
| cat ingestion_server/test/ingestion_logs.txt | |
| - name: Upload ingestion test logs | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ing_logs | |
| path: ingestion_server/test/ingestion_logs.txt | |
| ####### | |
| # API # | |
| ####### | |
| test-api: | |
| name: Run tests for the API | |
| if: | | |
| needs.get-changes.outputs.ingestion_server == 'true' || | |
| needs.get-changes.outputs.api == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run API tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server api | |
| - name: Start API, ingest and index test data | |
| run: just api/init | |
| - name: Run API tests | |
| run: just api/test | |
| - name: Print API test logs | |
| if: success() || failure() | |
| run: | | |
| just logs > api_logs | |
| cat api_logs | |
| - name: Upload API test logs | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api_logs | |
| path: api_logs | |
| django-checks: | |
| name: Run Django checks | |
| if: | | |
| needs.get-changes.outputs.api == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - build-images | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - check_django | |
| - validate_openapi | |
| - check_migrations | |
| - test_doc | |
| include: | |
| - name: check_django | |
| recipe: api/dj check | |
| - name: validate_openapi | |
| recipe: api/dj spectacular --format openapi-json --validate --file openapi.json | |
| - name: check_migrations | |
| recipe: api/dj makemigrations --check | |
| - name: test_doc | |
| recipe: api/doc-test | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run API tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server api | |
| - name: Run check recipe | |
| run: just ${{ matrix.recipe }} | |
| env: | |
| DC_USER: root | |
| - name: Upload schema | |
| if: matrix.name == 'test_doc' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openapi.json | |
| path: ./api/openapi.json | |
| # This job runs when `django-checks` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-django-checks: | |
| name: Run Django checks | |
| if: | | |
| !cancelled() && | |
| needs.django-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - django-checks | |
| strategy: | |
| matrix: | |
| name: | |
| - check_django | |
| - validate_openapi | |
| - check_migrations | |
| - test_doc | |
| steps: | |
| - name: Pass | |
| run: echo 'Django checks are skipped because API is unchanged.' | |
| ############ | |
| # Frontend # | |
| ############ | |
| nuxt-build: | |
| name: Check Nuxt build | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run build | |
| run: just frontend/run build | |
| nuxt-checks: | |
| name: Run Nuxt checks | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - unit_test | |
| include: | |
| - name: unit_test | |
| script: "test:unit" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run Nuxt checks | |
| run: just frontend/run ${{ matrix.script }} | |
| # This job runs when `nuxt-checks` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-nuxt-checks: | |
| name: Run Nuxt checks | |
| if: | | |
| !cancelled() && | |
| needs.nuxt-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nuxt-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - unit_test | |
| steps: | |
| - name: Pass | |
| run: echo 'Playwright tests are skipped because frontend is unchanged.' | |
| package-checks: | |
| name: Run checks for packages/* | |
| if: | | |
| needs.get-changes.outputs.packages == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - build | |
| - unit_test | |
| include: | |
| - name: build | |
| script: "build" | |
| - name: unit_test | |
| script: "test:unit" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run packages checks | |
| run: pnpm --filter ./packages/* run --aggregate-output ${{ matrix.script }} | |
| # This job runs when `package-checks` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-package-checks: | |
| name: Run checks for packages/* | |
| if: | | |
| !cancelled() && | |
| needs.package-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - package-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - build | |
| - unit_test | |
| steps: | |
| - name: Pass | |
| run: echo 'Checks for packages are skipped because packages are unchanged.' | |
| playwright: | |
| name: Run Playwright tests | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - playwright_vr | |
| - playwright_e2e | |
| - storybook | |
| include: | |
| - name: playwright_vr | |
| script: "test:playwright visual-regression" | |
| - name: playwright_e2e | |
| script: "test:playwright e2e" | |
| - name: storybook | |
| script: "test:storybook" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run Playwright tests | |
| run: just frontend/run ${{ matrix.script }} --workers=2 | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| id: test-results | |
| with: | |
| name: ${{ matrix.name }}_test_results | |
| path: frontend/test-results | |
| # This job runs when `playwright` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-playwright: | |
| name: Run Playwright tests | |
| if: | | |
| !cancelled() && | |
| needs.playwright.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - playwright | |
| strategy: | |
| matrix: | |
| name: | |
| - playwright_vr | |
| - playwright_e2e | |
| - storybook | |
| steps: | |
| - name: Pass | |
| run: echo 'Playwright tests are skipped because frontend is unchanged.' | |
| playwright-test-failure-comment: | |
| name: Post Playwright test debugging instructions | |
| if: | | |
| !cancelled() && | |
| github.event_name == 'pull_request' && | |
| needs.playwright.result != '' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - playwright | |
| steps: | |
| - uses: peter-evans/find-comment@v2 | |
| id: test-results-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: Playwright failure test results | |
| - uses: actions/github-script@v6 | |
| if: steps.test-results-comment.outputs.comment-id != 0 | |
| with: | |
| script: | | |
| await github.rest.issues.deleteComment({ | |
| repo: context.repo.repo, | |
| owner: context.repo.owner, | |
| comment_id: ${{ steps.test-results-comment.outputs.comment-id }} | |
| }) | |
| console.log('Deleted comment with ID ${{ steps.test-results-comment.outputs.comment-id }}') | |
| - name: Build help body | |
| if: needs.playwright.result == 'failure' | |
| id: help-body | |
| run: | | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) # Security hardening: https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#understanding-the-risk-of-script-injections | |
| MESSAGE=$(cat <<HEREDOC | |
| help_body<<$EOF | |
| **Playwright failure test results**: <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> | |
| It looks like some of the Playwright tests failed. You can download the trace output and image diffs for the failed tests under the "Artifacts" section in the above page. | |
| You can read the [docs on how to use this artifact](https://docs.openverse.org/frontend/reference/playwright_tests.html#debugging). | |
| If the test is flaky, follow the [flaky test triage procedure](https://docs.openverse.org/general/test.html#flaky-tests). | |
| $EOF | |
| HEREDOC | |
| ) | |
| echo "$MESSAGE" | tee "$GITHUB_OUTPUT" | |
| - uses: peter-evans/create-or-update-comment@v3 | |
| id: create-comment | |
| # Do not leave a comment on forks | |
| if: | | |
| needs.playwright.result == 'failure' && | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.owner.login == 'WordPress' && | |
| github.actor != 'dependabot[bot]' | |
| ) | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: ${{ steps.help-body.outputs.help_body }} | |
| ################# | |
| # Documentation # | |
| ################# | |
| build-docs: | |
| name: Build full-stack docs | |
| if: | | |
| needs.get-changes.outputs.documentation == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| - name: Compile documentation | |
| uses: ./.github/actions/build-docs | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # Docs will be located at `/tmp/docs`. | |
| - name: Upload documentation | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: /tmp/docs/ | |
| emit-docs: | |
| name: Emit full-stack docs | |
| # https://github.com/actions/runner/issues/491#issuecomment-850884422 | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| ( | |
| github.event_name == 'push' && | |
| github.repository == 'WordPress/openverse' | |
| ) || | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.owner.login == 'WordPress' && | |
| github.actor != 'dependabot[bot]' | |
| ) | |
| ) && | |
| needs.get-changes.outputs.documentation == 'true' && | |
| (needs.test-ing.result == 'success' || needs.test-ing.result == 'skipped') && | |
| (needs.test-api.result == 'success' || needs.test-api.result == 'skipped') && | |
| (needs.test-cat.result == 'success' || needs.test-cat.result == 'skipped') && | |
| (needs.playwright.result == 'success' || needs.playwright.result == 'skipped') && | |
| needs.build-docs.result == 'success' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - test-cat | |
| - test-ing | |
| - test-api | |
| - playwright | |
| - build-docs | |
| steps: | |
| - name: Download documentation | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: documentation | |
| path: /tmp/docs | |
| - name: Recreate working directory # to avoid superfluous files from getting tracked automatically | |
| run: | | |
| cd .. | |
| sudo rm -rf openverse | |
| mkdir openverse | |
| - name: Checkout repository at `gh-pages` branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| - name: Checkout automations from repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: automation-checkout | |
| - name: Copy existing previews | |
| if: github.event_name == 'push' | |
| run: | | |
| mv /tmp/docs /tmp/gh-pages | |
| mv gh-pages/_preview /tmp/gh-pages/_preview | |
| - name: Replace preview of current PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| cp -r gh-pages /tmp/gh-pages | |
| sudo rm -rf /tmp/gh-pages/_preview/${{ github.event.pull_request.number }} | |
| mv /tmp/docs /tmp/gh-pages/_preview/${{ github.event.pull_request.number }} | |
| - name: Determine which files have changed | |
| if: github.event_name == 'pull_request' | |
| id: preview_diff | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PYTHONPATH: ${{ github.workspace }}/automation-checkout/automations/python | |
| working-directory: ${{ github.workspace }}/automation-checkout/automations/python/workflows | |
| run: python get_folder_differences.py | |
| - name: Deploy | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: /tmp/gh-pages | |
| force_orphan: true | |
| cname: docs.openverse.org | |
| - uses: peter-evans/find-comment@v2 | |
| if: github.event_name == 'pull_request' | |
| id: final-preview-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: Full-stack documentation | |
| - uses: peter-evans/create-or-update-comment@v3 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-id: ${{ steps.final-preview-comment.outputs.comment-id }} | |
| edit-mode: replace | |
| body: ${{ steps.preview_diff.outputs.body }} | |
| - name: Checkout repository # again, to enable cleaning | |
| uses: actions/checkout@v4 | |
| if: success() || failure() | |
| ################# | |
| # Docker images # | |
| ################# | |
| publish-images: | |
| name: Publish Docker images | |
| runs-on: ubuntu-latest | |
| # prevent running on fork PRs | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.determine-images.outputs.do_publish == 'true' && | |
| (needs.test-ing.result == 'success' || needs.test-ing.result == 'skipped') && | |
| (needs.test-api.result == 'success' || needs.test-api.result == 'skipped') && | |
| (needs.test-cat.result == 'success' || needs.test-cat.result == 'skipped') && | |
| (needs.playwright.result == 'success' || needs.playwright.result == 'skipped') | |
| needs: | |
| - determine-images | |
| - get-image-tag | |
| - build-images | |
| - test-ing # test for ingestion server | |
| - test-api # test for API | |
| - test-cat # test for catalog | |
| - playwright # test for frontend | |
| permissions: | |
| packages: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-images.outputs.publish_matrix) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Docker Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: https://ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Load Docker image `${{ matrix.image }}` | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: ${{ matrix.image }} | |
| - name: Load and tag image `${{ matrix.image }}` (latest & sha) | |
| run: | | |
| docker tag openverse-${{ matrix.image }} \ | |
| ghcr.io/wordpress/openverse-${{ matrix.image }}:latest | |
| docker tag openverse-${{ matrix.image }} \ | |
| ghcr.io/wordpress/openverse-${{ matrix.image }}:${{ needs.get-image-tag.outputs.image_tag }} | |
| docker push --all-tags ghcr.io/wordpress/openverse-${{ matrix.image }} | |
| ############## | |
| # Deployment # | |
| ############## | |
| # See https://github.com/WordPress/openverse/issues/1033 for why | |
| # we don't use the standard reusable workflow approach for these. | |
| deploy-frontend: | |
| name: Deploy staging frontend | |
| runs-on: ubuntu-latest | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.get-changes.outputs.frontend == 'true' && | |
| needs.playwright.result == 'success' && | |
| needs.publish-images.result == 'success' | |
| needs: | |
| - get-changes | |
| - playwright | |
| - get-image-tag | |
| - publish-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Deploy staging frontend | |
| uses: felixp8/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse-infrastructure | |
| token: ${{ secrets.ACCESS_TOKEN }} | |
| event_type: deploy_staging_nuxt | |
| client_payload: | | |
| { | |
| "actor": "${{ github.actor }}", | |
| "tag": "${{ needs.get-image-tag.outputs.image_tag }}", | |
| "run_name": "${{ steps.commit.outputs.commit_message }}" | |
| } | |
| wait_time: 60 # check every minute | |
| max_time: 1800 # allow up to 30 minutes for a deployment | |
| deploy-api: | |
| name: Deploy staging API | |
| runs-on: ubuntu-latest | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.get-changes.outputs.api == 'true' && | |
| needs.publish-images.result == 'success' | |
| needs: | |
| - get-changes | |
| - get-image-tag | |
| - publish-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Deploy staging API | |
| uses: felixp8/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse-infrastructure | |
| token: ${{ secrets.ACCESS_TOKEN }} | |
| event_type: deploy_staging_api | |
| client_payload: | | |
| { | |
| "actor": "${{ github.actor }}", | |
| "tag": "${{ needs.get-image-tag.outputs.image_tag }}", | |
| "run_name": "${{ steps.commit.outputs.commit_message }}" | |
| } | |
| wait_time: 60 # check every minute | |
| max_time: 1800 # allow up to 30 minutes for a deployment | |
| ################ | |
| # Notification # | |
| ################ | |
| send-report: | |
| name: Send Slack report | |
| runs-on: ubuntu-latest | |
| if: | | |
| !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| ( | |
| ((github.event_name == 'push' && needs.get-changes.outputs.documentation == 'true') && needs.emit-docs.result != 'success') || | |
| (needs.determine-images.outputs.do_publish == 'true' && needs.publish-images.result != 'success') || | |
| (needs.get-changes.outputs.frontend == 'true' && needs.deploy-frontend.result != 'success') || | |
| (needs.get-changes.outputs.api == 'true' && needs.deploy-api.result != 'success') | |
| ) | |
| needs: # the end products of the CI + CD workflow | |
| - get-changes | |
| - determine-images | |
| - emit-docs | |
| - publish-images | |
| - deploy-frontend | |
| - deploy-api | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Generate report | |
| id: report | |
| env: | |
| COMMIT_MESSAGE: ${{ steps.commit.outputs.commit_message }} | |
| DEPLOY_API_RESULT: ${{ needs.deploy-api.result }} | |
| DEPLOY_FRONTEND_RESULT: ${{ needs.deploy-frontend.result }} | |
| EMIT_DOCS_RESULT: ${{ needs.emit-docs.result }} | |
| GH_SLACK_USERNAME_MAP: ${{ secrets.GH_SLACK_USERNAME_MAP }} | |
| GITHUB_ACTOR: ${{ github.event.head_commit.author.username }} | |
| PUBLISH_IMAGES_RESULT: ${{ needs.publish-images.result }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| SERVER_URL: ${{ github.server_url }} | |
| working-directory: automations/python/workflows | |
| run: python generate_report.py | |
| - name: Send report | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: ${{ steps.report.outputs.payload }} | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_OV_ALERTS_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |