WIP Convert CircleCI config to Github Actions

.github/actions/health_check_action.yml

@@ -0,0 +1,16 @@
+name: 'URL Health Check'
+description: 'Performs URL health check with retries'
+inputs:
+  url:
+    description: 'Health check URL'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        for i in {1..20}; do
+          curl --fail -v "${{ inputs.url }}" && exit 0
+          sleep 5
+        done
+        exit 1

.github/not-on-master.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ "${GITHUB_REF}" == "master" ]; then
+  echo "Skipping this step on master..."
+else
+  exec "$@"
+fi

.github/workflows/build_test_deploy.yml

@@ -1,13 +1,217 @@
 name: CI Pipeline
 
 on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - '*'
   workflow_dispatch:
 
+env:
+  USER_NAME: root
+  USER_UID: 1000
+  USER_GID: 1000
+  DOCKER_USER: ${{ secrets.DOCKER_USER }}
+  DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+
 jobs:
-  foo:
+  static_frontend_code_checks:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout code
       uses: actions/checkout@v3
       with:
-        fetch-depth: 5
+        fetch-depth: 5
+
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+
+    - name: Install frontend dependencies
+      run: corepack enable && yarn install --immutable
+
+    - name: Lint frontend code and check formatting
+      run: yarn run check-frontend
+
+    - name: Typecheck frontend code
+      run: yarn typecheck
+
+    - name: Check for cyclic dependencies in frontend
+      run: yarn check-cyclic-dependencies
+
+
+  build_test_deploy:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 5
+
+      - name: "Custom environment variables"
+        run: |
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g')
+            echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV
+            DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}"
+            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          else
+            echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV
+            echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install frontend dependencies
+        run: corepack enable && yarn install --immutable
+
+      - name: Build webknossos (webpack)
+        run: yarn build
+
+      - name: Set up Java / SBT
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '21'
+          cache: 'sbt'
+
+      - name: Install dependencies
+        run: apt-get update \
+          && apt-get install -y \
+          findutils \
+          libdraco-dev \
+          libblosc1
+
+                    # build-essential \
+          # cmake \
+      # - name: Set up Docker
+      #   uses: docker/setup-buildx-action@v2
+
+      # - name: Build webknossos-dev docker image
+      #   run: |
+      #     docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
+      #     DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
+
+      # - name: Prepare dependency folders
+      #   run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+
+      # - name: Install frontend dependencies
+      #   run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean
+
+      # - name: Assert unique evolution numbers
+      #   run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+
+      # - name: Assert schema.sql and evolutions are equal
+      #   run: |
+      #     docker compose up -d postgres
+      #     sleep 3
+      #     docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
+
+      - name: Build webknossos (sbt)
+        run: |
+          if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            sbt -no-colors clean compile stage
+          else
+            sbt -no-colors -DfailOnWarning compile stage
+          fi
+
+      - name: Build webknossos-datastore (sbt)
+        run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
+
+      - name: Build webknossos-tracingstore (sbt)
+        run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
+
+      - name: Checksum App Dirs
+        run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
+
+      - name: Build webknossos docker image
+        run: |
+          docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
+          DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
+
+      - name: Build webknossos-datastore docker image
+        run: docker compose build --pull webknossos-datastore
+
+      - name: Build webknossos-tracingstore docker image
+        run: docker compose build --pull webknossos-tracingstore
+
+      - name: Run frontend tests
+        run: .github/not-on-master.sh docker compose run base yarn test-verbose
+
+      - name: Lint backend code and check formatting
+        run: .github/not-on-master.sh docker compose run backend-lint-format
+
+      - name: Run backend tests
+        run: .github/not-on-master.sh docker compose run backend-tests
+
+      - name: Run end-to-end tests
+        run: |
+          for i in {1..3}; do # retry
+            .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$?
+          done
+          (exit $s)
+
+      - name: Start webknossos, datastore, and tracingstore
+        run: |
+          docker compose up -d webknossos
+          docker compose up -d webknossos-datastore
+          docker compose up -d webknossos-tracingstore
+
+      - name: Run webknossos smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9000/api/health
+
+
+      - name: Run webknossos-datastore smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9090/data/health
+
+      - name: Run webknossos-tracingstore smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9050/tracings/health
+
+      - name: Stop webknossos, datastore, and tracingstore
+        run: docker compose down --volumes --remove-orphans
+
+      - name: Push docker images
+        run: |
+          function retry() {
+            for i in {1..5}; do
+              "$@" && s=0 && break || s=$?
+              sleep 10
+            done
+            return $s
+          }
+          retry docker login -u $DOCKER_USER -p $DOCKER_PASS
+          retry docker compose push webknossos
+          retry docker compose push webknossos-datastore
+          retry docker compose push webknossos-tracingstore
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+          fi
+          docker logout
+
+      - name: Build frontend documentation
+        run: |
+          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
+          yarn run docs --project-version $WK_VERSION
+
+
+      - name: Report coverage
+        run: .github/not-on-master.sh docker compose run base yarn coverage || true
+
+      # - name: Send Slack notification (master only)
+      #   run: .circleci/slack-notification.sh

Dockerfile

@@ -8,6 +8,7 @@ RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \
 RUN mkdir -p /webknossos
 WORKDIR /webknossos
 
+# Copy compiled Scala output from a previous build step, e.g. output of the Docker-dev image
 COPY target/universal/stage .
 
 RUN addgroup --system --gid 999 webknossos \

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "2.2"
-
 services:
   # Production
   webknossos:
@@ -105,8 +103,8 @@ services:
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root
-      - CIRCLE_TAG=${CIRCLE_TAG}
-      - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM}
+      - CI_TAG=${GITHUB_REF}
+      - CI_BUILD_NUM=${GITHUB_JOB}
       - NODE_OPTIONS=--max_old_space_size=2048
       # This will be picked up by ava so that the tests fail if a snapshot doesn't exist during CI, see https://github.com/avajs/ava/issues/1585
       - CI

project/BuildInfoSettings.scala

@@ -13,8 +13,8 @@ object BuildInfoSettings {
     }
 
   val ciBuild: String =
-    if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else ""
-  val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else ""
+    if (System.getenv().containsKey("CI_BUILD_NUM")) System.getenv().get("CI_BUILD_NUM") else ""
+  val ciTag: String = if (System.getenv().containsKey("CI_TAG")) System.getenv().get("CI_TAG") else ""
 
   def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "<getting commit hash failed>")
   def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "<getting git date failed>")

yarn.lock

@@ -8109,7 +8109,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash-es@npm:^4.17.21, lodash-es@npm:^4.2.1":
+"lodash-es@npm:^4.17.21":
   version: 4.17.21
   resolution: "lodash-es@npm:4.17.21"
   checksum: 10c0/fb407355f7e6cd523a9383e76e6b455321f0f153a6c9625e21a8827d10c54c2a2341bd2ae8d034358b60e07325e1330c14c224ff582d04612a46a4f0479ff2f2
@@ -8144,7 +8144,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21, lodash@npm:^4.2.1":
+"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21":
   version: 4.17.21
   resolution: "lodash@npm:4.17.21"
   checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c
@@ -11587,14 +11587,11 @@ __metadata:
   linkType: hard
 
 "redux@npm:^4.2.0":
-  version: 3.7.2
-  resolution: "redux@npm:3.7.2"
+  version: 4.2.1
+  resolution: "redux@npm:4.2.1"
   dependencies:
-    lodash: "npm:^4.2.1"
-    lodash-es: "npm:^4.2.1"
-    loose-envify: "npm:^1.1.0"
-    symbol-observable: "npm:^1.0.3"
-  checksum: 10c0/544456f95734de33326637b370894addb57d9de2524edf36a20e4a326d0a36a0e223979d027545c5aa8a8d7a2859363981f63d1146401b72df0d16f373dd09cb
+    "@babel/runtime": "npm:^7.9.2"
+  checksum: 10c0/136d98b3d5dbed1cd6279c8c18a6a74c416db98b8a432a46836bdd668475de6279a2d4fd9d1363f63904e00f0678a8a3e7fa532c897163340baf1e71bb42c742
   languageName: node
   linkType: hard
 
@@ -12835,13 +12832,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"symbol-observable@npm:^1.0.3":
-  version: 1.2.0
-  resolution: "symbol-observable@npm:1.2.0"
-  checksum: 10c0/009fee50798ef80ed4b8195048288f108b03de162db07493f2e1fd993b33fafa72d659e832b584da5a2427daa78e5a738fb2a9ab027ee9454252e0bedbcd1fdc
-  languageName: node
-  linkType: hard
-
 "symbol-tree@npm:^3.2.4":
   version: 3.2.4
   resolution: "symbol-tree@npm:3.2.4"