Start data refreshes on Mondays UTC 00:00 #5100

Workflow file for this run

	name: CI + CD

	on:
	pull_request:
	push:
	branches:
	- main
	workflow_dispatch:
	inputs:
	image_tag:
	description: "The tag to assign to the images built in the workflow."
	type: string
	required: true

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	# Don't continue building images for a PR if the PR is updated quickly
	# For other workflows, allow them to complete and just block on them. This
	# ensures deployments in particular to happen in series rather than parallel.
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	###########
	# Helpers #
	###########

	get-changes:
	name: Get changes
	runs-on: ubuntu-latest
	permissions:
	pull-requests: read
	outputs:
	changes: ${{ steps.paths-filter.outputs.changes }}
	catalog: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'catalog') }}
	ingestion_server: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ingestion_server') }}
	api: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'api') }}
	frontend: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'frontend') }}
	documentation: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'documentation') }}
	ci_cd: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ci_cd') }}
	packages: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'packages') }}
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Get changes
	id: paths-filter
	uses: ./.github/actions/get-changes

	get-image-tag:
	name: Get image tag
	runs-on: ubuntu-latest
	outputs:
	image_tag: ${{ steps.get-image-tag.outputs.image_tag }}

	steps:
	- name: Get image tag
	id: get-image-tag
	run: \|
	if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
	echo "image_tag=${{ inputs.image_tag }}" >> "$GITHUB_OUTPUT"
	else
	echo "image_tag=${{ github.sha }}" >> "$GITHUB_OUTPUT"
	fi

	determine-images:
	name: Determine images to build and publish
	runs-on: ubuntu-latest
	outputs:
	do_build: ${{ steps.set-matrix.outputs.do_build }}
	build_matrix: ${{ steps.set-matrix.outputs.build_matrix }}
	do_publish: ${{ steps.set-matrix.outputs.do_publish }}
	publish_matrix: ${{ steps.set-matrix.outputs.publish_matrix }}
	needs:
	- get-changes

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Set matrix images
	id: set-matrix
	env:
	CHANGES: ${{ needs.get-changes.outputs.changes }}
	PYTHONPATH: ${{ github.workspace }}/automations/python
	working-directory: automations/python/workflows
	run: python set_matrix_images.py

	#############
	# Universal #
	#############

	lint: # This includes type-checking of the frontend.
	name: Lint files
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	# No Python setup is needed, the built-in Python runtime is okay.
	setup_python: false
	# Node.js is needed by lint actions.
	install_recipe: "node-install"

	- name: Cache pre-commit envs
	uses: actions/cache@v3
	with:
	path: ~/.cache/pre-commit
	key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}

	- name: Run pre-commit to lint files
	run: \|
	just precommit
	just lint

	add-stack-label:
	name: Add stack label
	if: \|
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.owner.login == 'WordPress' &&
	github.actor != 'dependabot[bot]'
	runs-on: ubuntu-latest
	needs:
	- get-changes

	steps:
	- name: Apply stack labels
	uses: actions/github-script@v6
	with:
	# Also see
	# - list of stack filters: `.github/filters.yml`
	# - list of stack labels: https://github.com/WordPress/openverse/labels?q=stack
	script: \|
	const stacks = ["catalog", "api", "ingestion_server", "frontend", "documentation", "mgmt"]
	const labels = JSON
	.parse('${{ needs.get-changes.outputs.changes }}')
	.filter(change => stacks.includes(change))
	.map(change => `🧱 stack: ${change.replace("_", " ")}`)
	if (!labels.length) {
	labels.push("🚦 status: awaiting triage", "🏷 status: label work required")
	}
	github.rest.issues.addLabels({
	issue_number: context.issue.number,
	owner: context.repo.owner,
	repo: context.repo.repo,
	labels,
	})

	build-images:
	name: Build Docker images
	if: needs.determine-images.outputs.do_build == 'true'
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(needs.determine-images.outputs.build_matrix) }}
	needs:
	- get-image-tag
	- lint
	- determine-images

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	# Sets up `just` and Node.js.
	# Node.js is needed for the frontend to download translations.
	# `just` is needed to run the recipes and set build args.
	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	setup_nodejs: ${{ matrix.image == 'frontend' && 'true' \|\| 'false' }}
	install_recipe: ${{ matrix.image == 'frontend' && 'node-install' \|\| '' }}

	# Sets build args specifying versions needed to build Docker image.
	- name: Prepare build args
	id: prepare-build-args
	run: \|
	just versions
	just versions >> "$GITHUB_OUTPUT"

	# ℹ️Step only applies for frontend image.
	# This step
	# - downloads translation strings from GlotPress so that they can be
	# bundled inside the Docker image
	- name: Prepare frontend for building
	if: matrix.image == 'frontend'
	run: \|
	just frontend/run i18n

	- name: Setup Docker Buildx
	uses: docker/setup-buildx-action@v2
	with:
	install: true

	- name: Build image `${{ matrix.image }}`
	uses: docker/build-push-action@v4
	with:
	context: ${{ matrix.context \|\| matrix.image }}
	target: ${{ matrix.target }}
	push: false
	tags: openverse-${{ matrix.image }}
	cache-from: type=gha,scope=${{ matrix.image }}
	cache-to: type=gha,scope=${{ matrix.image }}
	outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar
	build-contexts: ${{ matrix.build-contexts \|\| '' }}
	build-args: \|
	SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }}
	CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }}
	CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }}
	API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }}
	INGESTION_PY_VERSION=${{ steps.prepare-build-args.outputs.ingestion_py_version }}
	FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }}
	FRONTEND_PNPM_VERSION=${{ steps.prepare-build-args.outputs.frontend_pnpm_version }}

	- name: Upload image `${{ matrix.image }}`
	uses: actions/upload-artifact@v3
	with:
	name: ${{ matrix.image }}
	path: /tmp/${{ matrix.image }}.tar

	###########
	# Catalog #
	###########

	test-cat:
	name: Run tests for the catalog
	if: \|
	needs.get-changes.outputs.catalog == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 15
	needs:
	- get-changes
	- build-images

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	# Python is not needed to run the tests.
	setup_nodejs: false
	# Node.js is not needed to run the tests.
	install_recipe: ""

	- name: Load Docker images
	uses: ./.github/actions/load-img
	with:
	setup_images: upstream_db catalog

	- name: Run tests
	run: \|
	just catalog/test --extended

	catalog-checks:
	name: Run catalog checks
	if: \|
	needs.get-changes.outputs.catalog == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- build-images

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	# Python is not needed to run the tests.
	setup_python: false
	# Node.js is needed by lint actions.
	install_recipe: "node-install"

	- name: Load Docker images
	uses: ./.github/actions/load-img
	with:
	setup_images: upstream_db catalog

	- name: Check DAG documentation
	run: \|
	just catalog/generate-dag-docs true

	####################
	# Ingestion server #
	####################

	test-ing:
	name: Run tests for ingestion-server
	if: \|
	needs.get-changes.outputs.ingestion_server == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 15
	needs:
	- get-changes
	- build-images

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	# Python is needed to run the test.
	setup_nodejs: false
	# Node.js is not needed to run ingestion server tests.
	install_recipe: ingestion_server/install

	- name: Load Docker images
	uses: ./.github/actions/load-img
	with:
	setup_images: upstream_db ingestion_server

	- name: Run ingestion-server tests
	run: just ingestion_server/test-local

	- name: Print ingestion-server test logs
	run: \|
	just ingestion_server/test-logs > ingestion_server/test/ingestion_logs.txt
	cat ingestion_server/test/ingestion_logs.txt

	- name: Upload ingestion test logs
	if: success() \|\| failure()
	uses: actions/upload-artifact@v3
	with:
	name: ing_logs
	path: ingestion_server/test/ingestion_logs.txt

	#######
	# API #
	#######

	test-api:
	name: Run tests for the API
	if: \|
	needs.get-changes.outputs.ingestion_server == 'true' \|\|
	needs.get-changes.outputs.api == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 15
	needs:
	- get-changes
	- build-images

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	# Python is not needed to run the tests.
	setup_nodejs: false
	# Node.js is not needed to run API tests.
	install_recipe: ""

	- name: Load Docker images
	uses: ./.github/actions/load-img
	with:
	setup_images: upstream_db ingestion_server api

	- name: Start API, ingest and index test data
	run: just api/init

	- name: Run API tests
	run: just api/test

	- name: Print API test logs
	if: success() \|\| failure()
	run: \|
	just logs > api_logs
	cat api_logs

	- name: Upload API test logs
	if: success() \|\| failure()
	uses: actions/upload-artifact@v3
	with:
	name: api_logs
	path: api_logs

	django-checks:
	name: Run Django checks
	if: \|
	needs.get-changes.outputs.api == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- build-images
	strategy:
	fail-fast: false
	matrix:
	name:
	- check_django
	- validate_openapi
	- check_migrations
	- test_doc
	include:
	- name: check_django
	recipe: api/dj check
	- name: validate_openapi
	recipe: api/dj spectacular --format openapi-json --validate --file openapi.json
	- name: check_migrations
	recipe: api/dj makemigrations --check --noinput --merge
	- name: test_doc
	recipe: api/doc-test

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	# Python is not needed to run the tests.
	setup_nodejs: false
	# Node.js is not needed to run API tests.
	install_recipe: ""

	- name: Load Docker images
	uses: ./.github/actions/load-img
	with:
	setup_images: upstream_db ingestion_server api

	- name: Run check recipe
	run: just ${{ matrix.recipe }}
	env:
	DC_USER: root

	- name: Upload schema
	if: matrix.name == 'test_doc'
	uses: actions/upload-artifact@v3
	with:
	name: openapi.json
	path: ./api/openapi.json

	# This job runs when `django-checks` doesn't and always passes, thus allowing
	# PRs to meet the required checks criteria and be merged.
	bypass-django-checks:
	name: Run Django checks
	if: \|
	!cancelled() &&
	needs.django-checks.result == 'skipped'
	runs-on: ubuntu-latest
	needs:
	- django-checks
	strategy:
	matrix:
	name:
	- check_django
	- validate_openapi
	- check_migrations
	- test_doc

	steps:
	- name: Pass
	run: echo 'Django checks are skipped because API is unchanged.'

	############
	# Frontend #
	############

	nuxt-build:
	name: Check Nuxt build
	if: \|
	needs.get-changes.outputs.frontend == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- lint

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	install_recipe: node-install

	- name: Run build
	run: just frontend/run build

	nuxt-checks:
	name: Run Nuxt checks
	if: \|
	needs.get-changes.outputs.frontend == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- lint
	strategy:
	fail-fast: false
	matrix:
	name:
	- unit_test
	- storybook_smoke_test
	include:
	- name: unit_test
	script: "test:unit"
	- name: storybook_smoke_test
	script: "storybook:smoke"

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	install_recipe: node-install

	- name: Run Nuxt checks
	run: just frontend/run ${{ matrix.script }}

	# This job runs when `nuxt-checks` doesn't and always passes, thus allowing
	# PRs to meet the required checks criteria and be merged.
	bypass-nuxt-checks:
	name: Run Nuxt checks
	if: \|
	!cancelled() &&
	needs.nuxt-checks.result == 'skipped'
	runs-on: ubuntu-latest
	needs:
	- nuxt-checks
	strategy:
	fail-fast: false
	matrix:
	name:
	- unit_test
	- storybook_smoke_test

	steps:
	- name: Pass
	run: echo 'Playwright tests are skipped because frontend is unchanged.'

	package-checks:
	name: Run checks for packages/*
	if: \|
	needs.get-changes.outputs.packages == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- lint
	strategy:
	fail-fast: false
	matrix:
	name:
	- build
	- unit_test
	include:
	- name: build
	script: "build"
	- name: unit_test
	script: "test:unit"

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	install_recipe: node-install

	- name: Run packages checks
	run: pnpm --filter ./packages/* run --aggregate-output ${{ matrix.script }}

	# This job runs when `package-checks` doesn't and always passes, thus allowing
	# PRs to meet the required checks criteria and be merged.
	bypass-package-checks:
	name: Run checks for packages/*
	if: \|
	!cancelled() &&
	needs.package-checks.result == 'skipped'
	runs-on: ubuntu-latest
	needs:
	- package-checks
	strategy:
	fail-fast: false
	matrix:
	name:
	- build
	- unit_test

	steps:
	- name: Pass
	run: echo 'Checks for packages are skipped because packages are unchanged.'

	playwright:
	name: Run Playwright tests
	if: \|
	needs.get-changes.outputs.frontend == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 15
	needs:
	- get-changes
	- lint
	strategy:
	fail-fast: false
	matrix:
	name:
	- playwright_vr
	- playwright_e2e
	- storybook
	include:
	- name: playwright_vr
	script: "test:playwright visual-regression"
	- name: playwright_e2e
	script: "test:playwright e2e"
	- name: storybook
	script: "test:storybook"

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env
	with:
	setup_python: false
	install_recipe: node-install

	- name: Run Playwright tests
	run: just frontend/run ${{ matrix.script }} --workers=2

	- uses: actions/upload-artifact@v3
	if: failure()
	id: test-results
	with:
	name: ${{ matrix.name }}_test_results
	path: frontend/test-results

	# This job runs when `playwright` doesn't and always passes, thus allowing
	# PRs to meet the required checks criteria and be merged.
	bypass-playwright:
	name: Run Playwright tests
	if: \|
	!cancelled() &&
	needs.playwright.result == 'skipped'
	runs-on: ubuntu-latest
	needs:
	- playwright
	strategy:
	matrix:
	name:
	- playwright_vr
	- playwright_e2e
	- storybook

	steps:
	- name: Pass
	run: echo 'Playwright tests are skipped because frontend is unchanged.'

	playwright-test-failure-comment:
	name: Post Playwright test debugging instructions
	if: \|
	!cancelled() &&
	github.event_name == 'pull_request' &&
	needs.playwright.result != ''
	runs-on: ubuntu-latest
	needs:
	- playwright

	steps:
	- uses: peter-evans/find-comment@v2
	id: test-results-comment
	with:
	issue-number: ${{ github.event.pull_request.number }}
	body-includes: Playwright failure test results

	- uses: actions/github-script@v6
	if: steps.test-results-comment.outputs.comment-id != 0
	with:
	script: \|
	await github.rest.issues.deleteComment({
	repo: context.repo.repo,
	owner: context.repo.owner,
	comment_id: ${{ steps.test-results-comment.outputs.comment-id }}
	})
	console.log('Deleted comment with ID ${{ steps.test-results-comment.outputs.comment-id }}')

	- name: Build help body
	if: needs.playwright.result == 'failure'
	id: help-body
	run: \|
	EOF=$(dd if=/dev/urandom bs=15 count=1 status=none \| base64) # Security hardening: https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#understanding-the-risk-of-script-injections
	MESSAGE=$(cat <<HEREDOC
	help_body<<$EOF
	Playwright failure test results: <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}>

	It looks like some of the Playwright tests failed. You can download the trace output and image diffs for the failed tests under the "Artifacts" section in the above page.

	You can read the [docs on how to use this artifact](https://docs.openverse.org/frontend/reference/playwright_tests.html#debugging).

	If the test is flaky, follow the [flaky test triage procedure](https://docs.openverse.org/general/test.html#flaky-tests).
	$EOF
	HEREDOC
	)
	echo "$MESSAGE"
	echo "$MESSAGE" >> "$GITHUB_OUTPUT"

	- uses: peter-evans/create-or-update-comment@v3
	id: create-comment
	# Do not leave a comment on forks
	if: \|
	needs.playwright.result == 'failure' &&
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.owner.login == 'WordPress' &&
	github.actor != 'dependabot[bot]'
	)
	with:
	issue-number: ${{ github.event.pull_request.number }}
	body: ${{ steps.help-body.outputs.help_body }}

	#################
	# Documentation #
	#################

	build-docs:
	name: Build full-stack docs
	if: \|
	needs.get-changes.outputs.documentation == 'true' \|\|
	needs.get-changes.outputs.ci_cd == 'true'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- lint

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Setup CI env
	uses: ./.github/actions/setup-env

	- name: Compile documentation
	uses: ./.github/actions/build-docs
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	# Docs will be located at `/tmp/docs`.

	- name: Upload documentation
	uses: actions/upload-artifact@v3
	with:
	name: documentation
	path: /tmp/docs/

	emit-docs:
	name: Emit full-stack docs
	# https://github.com/actions/runner/issues/491#issuecomment-850884422
	if: \|
	!failure() && !cancelled() &&
	(
	(
	github.event_name == 'push' &&
	github.repository == 'WordPress/openverse'
	) \|\|
	(
	github.event_name == 'pull_request' &&
	github.event.pull_request.head.repo.owner.login == 'WordPress' &&
	github.actor != 'dependabot[bot]'
	)
	) &&
	needs.get-changes.outputs.documentation == 'true' &&
	(needs.test-ing.result == 'success' \|\| needs.test-ing.result == 'skipped') &&
	(needs.test-api.result == 'success' \|\| needs.test-api.result == 'skipped') &&
	(needs.test-cat.result == 'success' \|\| needs.test-cat.result == 'skipped') &&
	(needs.playwright.result == 'success' \|\| needs.playwright.result == 'skipped') &&
	needs.build-docs.result == 'success'
	runs-on: ubuntu-latest
	needs:
	- get-changes
	- test-cat
	- test-ing
	- test-api
	- playwright
	- build-docs

	steps:
	- name: Download documentation
	uses: actions/download-artifact@v3
	with:
	name: documentation
	path: /tmp/docs

	- name: Recreate working directory # to avoid superfluous files from getting tracked automatically
	run: \|
	cd ..
	sudo rm -rf openverse
	mkdir openverse

	- name: Checkout repository at `gh-pages` branch
	uses: actions/checkout@v3
	with:
	ref: gh-pages
	path: gh-pages

	- name: Checkout automations from repository
	uses: actions/checkout@v3
	with:
	path: automation-checkout

	- name: Copy existing previews
	if: github.event_name == 'push'
	run: \|
	mv /tmp/docs /tmp/gh-pages
	mv gh-pages/_preview /tmp/gh-pages/_preview

	- name: Replace preview of current PR
	if: github.event_name == 'pull_request'
	run: \|
	cp -r gh-pages /tmp/gh-pages
	sudo rm -rf /tmp/gh-pages/_preview/${{ github.event.pull_request.number }}
	mv /tmp/docs /tmp/gh-pages/_preview/${{ github.event.pull_request.number }}

	- name: Determine which files have changed
	if: github.event_name == 'pull_request'
	id: preview_diff
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	PYTHONPATH: ${{ github.workspace }}/automation-checkout/automations/python
	working-directory: ${{ github.workspace }}/automation-checkout/automations/python/workflows
	run: python get_folder_differences.py

	- name: Deploy
	uses: peaceiris/actions-gh-pages@v3
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	publish_dir: /tmp/gh-pages
	force_orphan: true
	cname: docs.openverse.org

	- uses: peter-evans/find-comment@v2
	if: github.event_name == 'pull_request'
	id: final-preview-comment
	with:
	issue-number: ${{ github.event.pull_request.number }}
	body-includes: Full-stack documentation

	- uses: peter-evans/create-or-update-comment@v3
	if: github.event_name == 'pull_request'
	with:
	issue-number: ${{ github.event.pull_request.number }}
	comment-id: ${{ steps.final-preview-comment.outputs.comment-id }}
	edit-mode: replace
	body: ${{ steps.preview_diff.outputs.body }}

	- name: Checkout repository # again, to enable cleaning
	uses: actions/checkout@v3
	if: success() \|\| failure()

	#################
	# Docker images #
	#################

	publish-images:
	name: Publish Docker images
	runs-on: ubuntu-latest
	# prevent running on fork PRs
	if: \|
	!failure() && !cancelled() &&
	github.event_name == 'push' &&
	github.repository == 'WordPress/openverse' &&
	needs.determine-images.outputs.do_publish == 'true' &&
	(needs.test-ing.result == 'success' \|\| needs.test-ing.result == 'skipped') &&
	(needs.test-api.result == 'success' \|\| needs.test-api.result == 'skipped') &&
	(needs.test-cat.result == 'success' \|\| needs.test-cat.result == 'skipped') &&
	(needs.playwright.result == 'success' \|\| needs.playwright.result == 'skipped')
	needs:
	- determine-images
	- get-image-tag
	- build-images
	- test-ing # test for ingestion server
	- test-api # test for API
	- test-cat # test for catalog
	- playwright # test for frontend
	permissions:
	packages: write
	contents: read
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(needs.determine-images.outputs.publish_matrix) }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Log in to GitHub Docker Registry
	uses: docker/login-action@v2
	with:
	registry: https://ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Load Docker image `${{ matrix.image }}`
	uses: ./.github/actions/load-img
	with:
	setup_images: ${{ matrix.image }}

	- name: Load and tag image `${{ matrix.image }}` (latest & sha)
	run: \|
	docker tag openverse-${{ matrix.image }} \
	ghcr.io/wordpress/openverse-${{ matrix.image }}:latest
	docker tag openverse-${{ matrix.image }} \
	ghcr.io/wordpress/openverse-${{ matrix.image }}:${{ needs.get-image-tag.outputs.image_tag }}
	docker push --all-tags ghcr.io/wordpress/openverse-${{ matrix.image }}

	##############
	# Deployment #
	##############

	# See https://github.com/WordPress/openverse/issues/1033 for why
	# we don't use the standard reusable workflow approach for these.

	deploy-frontend:
	name: Deploy staging frontend
	runs-on: ubuntu-latest
	if: \|
	!failure() && !cancelled() &&
	github.event_name == 'push' &&
	needs.get-changes.outputs.frontend == 'true' &&
	needs.playwright.result == 'success' &&
	needs.publish-images.result == 'success'
	needs:
	- get-changes
	- playwright
	- get-image-tag
	- publish-images
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Get commit message
	id: commit
	env:
	COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
	PYTHONPATH: ${{ github.workspace }}/automations/python
	working-directory: automations/python/workflows
	run: python get_commit_message.py

	- name: Deploy staging frontend
	uses: felixp8/[email protected]
	with:
	owner: WordPress
	repo: openverse-infrastructure
	token: ${{ secrets.ACCESS_TOKEN }}
	event_type: deploy_staging_nuxt
	client_payload: \|
	{
	"actor": "${{ github.actor }}",
	"tag": "${{ needs.get-image-tag.outputs.image_tag }}",
	"run_name": "${{ steps.commit.outputs.commit_message }}"
	}
	wait_time: 60 # check every minute
	max_time: 1800 # allow up to 30 minutes for a deployment

	deploy-api:
	name: Deploy staging API
	runs-on: ubuntu-latest
	if: \|
	!failure() && !cancelled() &&
	github.event_name == 'push' &&
	needs.get-changes.outputs.api == 'true' &&
	needs.publish-images.result == 'success'
	needs:
	- get-changes
	- get-image-tag
	- publish-images
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Get commit message
	id: commit
	env:
	COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
	PYTHONPATH: ${{ github.workspace }}/automations/python
	working-directory: automations/python/workflows
	run: python get_commit_message.py

	- name: Deploy staging API
	uses: felixp8/[email protected]
	with:
	owner: WordPress
	repo: openverse-infrastructure
	token: ${{ secrets.ACCESS_TOKEN }}
	event_type: deploy_staging_api
	client_payload: \|
	{
	"actor": "${{ github.actor }}",
	"tag": "${{ needs.get-image-tag.outputs.image_tag }}",
	"run_name": "${{ steps.commit.outputs.commit_message }}"
	}
	wait_time: 60 # check every minute
	max_time: 1800 # allow up to 30 minutes for a deployment

	- name: Deploy staging thumbnails
	uses: felixp8/[email protected]
	with:
	owner: WordPress
	repo: openverse-infrastructure
	token: ${{ secrets.ACCESS_TOKEN }}
	event_type: deploy_staging_api_thumbnails
	client_payload: \|
	{
	"actor": "${{ github.actor }}",
	"tag": "${{ needs.get-image-tag.outputs.image_tag }}",
	"run_name": "${{ steps.commit.outputs.commit_message }}"
	}
	wait_time: 60 # check every minute
	max_time: 1800 # allow up to 30 minutes for a deployment

	################
	# Notification #
	################

	send-report:
	name: Send Slack report
	runs-on: ubuntu-latest
	if: \|
	!cancelled() &&
	github.event_name == 'push' &&
	github.repository == 'WordPress/openverse' &&
	(
	(needs.get-changes.outputs.documentation == 'true' && needs.emit-docs.result != 'success') \|\|
	(needs.determine-images.outputs.do_publish == 'true' && needs.publish-images.result != 'success') \|\|
	(needs.get-changes.outputs.frontend == 'true' && needs.deploy-frontend.result != 'success') \|\|
	(needs.get-changes.outputs.api == 'true' && needs.deploy-api.result != 'success')
	)
	needs: # the end products of the CI + CD workflow
	- get-changes
	- determine-images
	- emit-docs
	- publish-images
	- deploy-frontend
	- deploy-api

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Get commit message
	id: commit
	env:
	COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
	PYTHONPATH: ${{ github.workspace }}/automations/python
	working-directory: automations/python/workflows
	run: python get_commit_message.py

	- name: Generate report
	id: report
	env:
	COMMIT_MESSAGE: ${{ steps.commit.outputs.commit_message }}
	DEPLOY_API_RESULT: ${{ needs.deploy-api.result }}
	DEPLOY_FRONTEND_RESULT: ${{ needs.deploy-frontend.result }}
	EMIT_DOCS_RESULT: ${{ needs.emit-docs.result }}
	GH_SLACK_USERNAME_MAP: ${{ secrets.GH_SLACK_USERNAME_MAP }}
	GITHUB_ACTOR: ${{ github.event.head_commit.author.username }}
	PUBLISH_IMAGES_RESULT: ${{ needs.publish-images.result }}
	PYTHONPATH: ${{ github.workspace }}/automations/python
	REPOSITORY: ${{ github.repository }}
	RUN_ID: ${{ github.run_id }}
	SERVER_URL: ${{ github.server_url }}
	working-directory: automations/python/workflows
	run: python generate_report.py

	- name: Send report
	uses: slackapi/[email protected]
	with:
	payload: ${{ steps.report.outputs.payload }}
	env:
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_OV_ALERTS_WEBHOOK_URL }}
	SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Start data refreshes on Mondays UTC 00:00 #5100

Workflow file

Start data refreshes on Mondays UTC 00:00 #5100

Jobs

Run details

Workflow file for this run