diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh new file mode 100644 index 0000000000..fc9ccc8fc0 --- /dev/null +++ b/.github/not-on-master.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "${GITHUB_REF}" == "master" ]; then + echo "Skipping this step on master..." +else + exec "$@" +fi diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 599e2b22e0..17f93d24d9 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1,13 +1,208 @@ name: CI Pipeline on: + push: + branches: + - '*' + pull_request: + branches: + - '*' workflow_dispatch: +env: + USER_NAME: circleci + USER_UID: 1000 + USER_GID: 1000 + TZ: Europe/Berlin + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} + jobs: - foo: + static_frontent_code_checks: runs-on: ubuntu-20.04 steps: - name: Checkout code uses: actions/checkout@v3 with: - fetch-depth: 5 \ No newline at end of file + fetch-depth: 5 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install frontend dependencies + run: corepack enable && yarn install --immutable + + - name: Lint frontend code and check formatting + run: yarn run check-frontend + + - name: Typecheck frontend code + run: yarn typecheck + + - name: Check for cyclic dependencies in frontend + run: yarn check-cyclic-dependencies + + + + + build_test_deploy: + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 5 + + - name: "Custom environment variables" + run: | + if [[ ${{ github.ref_type }} == "branch" ]]; then + NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g') + echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV + DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}" + echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV + else + echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV + echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV + fi + + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + + - name: Build webknossos-dev docker image + run: | + docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true + DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base + + - name: Prepare dependency folders + run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + + - name: Install frontend dependencies + run: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable + + - name: Assert unique evolution numbers + run: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers + + - name: Assert schema.sql and evolutions are equal + run: | + docker-compose up -d postgres + sleep 3 + docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema + + - name: Build frontend documentation + run: | + WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} + docker-compose run base yarn run docs --project-version $WK_VERSION + + - name: Build webknossos (webpack) + run: docker-compose run base yarn build + + - name: Build webknossos (sbt) + run: | + if [ "${{ github.ref }}" == "refs/heads/master" ]; then + docker-compose run compile sbt -no-colors clean compile stage + else + docker-compose run compile sbt -no-colors -DfailOnWarning compile stage + fi + + - name: Build webknossos-datastore (sbt) + run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + + - name: Build webknossos-tracingstore (sbt) + run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + + - name: Checksum App Dirs + run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt + + - name: Build webknossos docker image + run: | + docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true + DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos + + - name: Build webknossos-datastore docker image + run: docker-compose build --pull webknossos-datastore + + - name: Build webknossos-tracingstore docker image + run: docker-compose build --pull webknossos-tracingstore + + - name: Run frontend tests + run: .github/not-on-master.sh docker-compose run base yarn test-verbose + + - name: Lint backend code and check formatting + run: .github/not-on-master.sh docker-compose run backend-lint-format + + - name: Run backend tests + run: .github/not-on-master.sh docker-compose run backend-tests + + - name: Run end-to-end tests + run: | + for i in {1..3}; do # retry + .github/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? + done + (exit $s) + + - name: Validate frontend types + run: .github/not-on-master.sh docker-compose run base yarn typecheck + + - name: Start webknossos, datastore, and tracingstore + run: | + docker-compose up -d webknossos + docker-compose up -d webknossos-datastore + docker-compose up -d webknossos-tracingstore + + - name: Run webknossos smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Run webknossos-datastore smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Run webknossos-tracingstore smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Stop webknossos, datastore, and tracingstore + run: docker-compose down --volumes --remove-orphans + + - name: Push docker images + run: | + function retry() { + for i in {1..5}; do + "$@" && s=0 && break || s=$? + sleep 10 + done + return $s + } + retry docker login -u $DOCKER_USER -p $DOCKER_PASS + retry docker-compose push webknossos + retry docker-compose push webknossos-datastore + retry docker-compose push webknossos-tracingstore + if [[ ${{ github.ref_type }} == "branch" ]]; then + docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + fi + docker logout + + - name: Report coverage + run: .github/not-on-master.sh docker-compose run base yarn coverage || true + + # - name: Send Slack notification (master only) + # run: .circleci/slack-notification.sh