From 69a4e4c117f0098143374f424537efda35b82fb9 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Thu, 24 Oct 2024 16:00:18 +0200 Subject: [PATCH 01/43] convert circle ci config to github actions --- .github/not-on-master.sh | 8 + .github/workflows/build_test_deploy.yml | 199 +++++++++++++++++++++++- 2 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 .github/not-on-master.sh diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh new file mode 100644 index 0000000000..fc9ccc8fc0 --- /dev/null +++ b/.github/not-on-master.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "${GITHUB_REF}" == "master" ]; then + echo "Skipping this step on master..." +else + exec "$@" +fi diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 599e2b22e0..17f93d24d9 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1,13 +1,208 @@ name: CI Pipeline on: + push: + branches: + - '*' + pull_request: + branches: + - '*' workflow_dispatch: +env: + USER_NAME: circleci + USER_UID: 1000 + USER_GID: 1000 + TZ: Europe/Berlin + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} + jobs: - foo: + static_frontent_code_checks: runs-on: ubuntu-20.04 steps: - name: Checkout code uses: actions/checkout@v3 with: - fetch-depth: 5 \ No newline at end of file + fetch-depth: 5 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install frontend dependencies + run: corepack enable && yarn install --immutable + + - name: Lint frontend code and check formatting + run: yarn run check-frontend + + - name: Typecheck frontend code + run: yarn typecheck + + - name: Check for cyclic dependencies in frontend + run: yarn check-cyclic-dependencies + + + + + build_test_deploy: + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 5 + + - name: "Custom environment variables" + run: | + if [[ ${{ github.ref_type }} == "branch" ]]; then + NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g') + echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV + DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}" + echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV + else + echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV + echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV + fi + + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + + - name: Build webknossos-dev docker image + run: | + docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true + DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base + + - name: Prepare dependency folders + run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + + - name: Install frontend dependencies + run: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable + + - name: Assert unique evolution numbers + run: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers + + - name: Assert schema.sql and evolutions are equal + run: | + docker-compose up -d postgres + sleep 3 + docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema + + - name: Build frontend documentation + run: | + WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} + docker-compose run base yarn run docs --project-version $WK_VERSION + + - name: Build webknossos (webpack) + run: docker-compose run base yarn build + + - name: Build webknossos (sbt) + run: | + if [ "${{ github.ref }}" == "refs/heads/master" ]; then + docker-compose run compile sbt -no-colors clean compile stage + else + docker-compose run compile sbt -no-colors -DfailOnWarning compile stage + fi + + - name: Build webknossos-datastore (sbt) + run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + + - name: Build webknossos-tracingstore (sbt) + run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + + - name: Checksum App Dirs + run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt + + - name: Build webknossos docker image + run: | + docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true + DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos + + - name: Build webknossos-datastore docker image + run: docker-compose build --pull webknossos-datastore + + - name: Build webknossos-tracingstore docker image + run: docker-compose build --pull webknossos-tracingstore + + - name: Run frontend tests + run: .github/not-on-master.sh docker-compose run base yarn test-verbose + + - name: Lint backend code and check formatting + run: .github/not-on-master.sh docker-compose run backend-lint-format + + - name: Run backend tests + run: .github/not-on-master.sh docker-compose run backend-tests + + - name: Run end-to-end tests + run: | + for i in {1..3}; do # retry + .github/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? + done + (exit $s) + + - name: Validate frontend types + run: .github/not-on-master.sh docker-compose run base yarn typecheck + + - name: Start webknossos, datastore, and tracingstore + run: | + docker-compose up -d webknossos + docker-compose up -d webknossos-datastore + docker-compose up -d webknossos-tracingstore + + - name: Run webknossos smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Run webknossos-datastore smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Run webknossos-tracingstore smoke test + run: | + for i in {1..20}; do # retry + curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$? + sleep 5 + done + (exit $s) + + - name: Stop webknossos, datastore, and tracingstore + run: docker-compose down --volumes --remove-orphans + + - name: Push docker images + run: | + function retry() { + for i in {1..5}; do + "$@" && s=0 && break || s=$? + sleep 10 + done + return $s + } + retry docker login -u $DOCKER_USER -p $DOCKER_PASS + retry docker-compose push webknossos + retry docker-compose push webknossos-datastore + retry docker-compose push webknossos-tracingstore + if [[ ${{ github.ref_type }} == "branch" ]]; then + docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + fi + docker logout + + - name: Report coverage + run: .github/not-on-master.sh docker-compose run base yarn coverage || true + + # - name: Send Slack notification (master only) + # run: .circleci/slack-notification.sh From 4a9811eff7c14db0c0b99cbd77d3bd6459892cfc Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Thu, 24 Oct 2024 16:03:40 +0200 Subject: [PATCH 02/43] use new docker compose syntax --- .github/workflows/build_test_deploy.yml | 54 ++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 17f93d24d9..506cf1ad82 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -71,44 +71,44 @@ jobs: - name: Build webknossos-dev docker image run: | docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true - DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base + DEV_CACHE=$NORMALIZED_BRANCH docker compose build base - name: Prepare dependency folders run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier - name: Install frontend dependencies - run: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable + run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable - name: Assert unique evolution numbers - run: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers + run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers - name: Assert schema.sql and evolutions are equal run: | - docker-compose up -d postgres + docker compose up -d postgres sleep 3 - docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema + docker compose run compile tools/postgres/dbtool.js check-evolutions-schema - name: Build frontend documentation run: | WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} - docker-compose run base yarn run docs --project-version $WK_VERSION + docker compose run base yarn run docs --project-version $WK_VERSION - name: Build webknossos (webpack) - run: docker-compose run base yarn build + run: docker compose run base yarn build - name: Build webknossos (sbt) run: | if [ "${{ github.ref }}" == "refs/heads/master" ]; then - docker-compose run compile sbt -no-colors clean compile stage + docker compose run compile sbt -no-colors clean compile stage else - docker-compose run compile sbt -no-colors -DfailOnWarning compile stage + docker compose run compile sbt -no-colors -DfailOnWarning compile stage fi - name: Build webknossos-datastore (sbt) - run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage - name: Build webknossos-tracingstore (sbt) - run: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage - name: Checksum App Dirs run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt @@ -116,38 +116,38 @@ jobs: - name: Build webknossos docker image run: | docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true - DEV_CACHE=$NORMALIZED_BRANCH docker-compose build --pull webknossos + DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos - name: Build webknossos-datastore docker image - run: docker-compose build --pull webknossos-datastore + run: docker compose build --pull webknossos-datastore - name: Build webknossos-tracingstore docker image - run: docker-compose build --pull webknossos-tracingstore + run: docker compose build --pull webknossos-tracingstore - name: Run frontend tests - run: .github/not-on-master.sh docker-compose run base yarn test-verbose + run: .github/not-on-master.sh docker compose run base yarn test-verbose - name: Lint backend code and check formatting - run: .github/not-on-master.sh docker-compose run backend-lint-format + run: .github/not-on-master.sh docker compose run backend-lint-format - name: Run backend tests - run: .github/not-on-master.sh docker-compose run backend-tests + run: .github/not-on-master.sh docker compose run backend-tests - name: Run end-to-end tests run: | for i in {1..3}; do # retry - .github/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? + .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$? done (exit $s) - name: Validate frontend types - run: .github/not-on-master.sh docker-compose run base yarn typecheck + run: .github/not-on-master.sh docker compose run base yarn typecheck - name: Start webknossos, datastore, and tracingstore run: | - docker-compose up -d webknossos - docker-compose up -d webknossos-datastore - docker-compose up -d webknossos-tracingstore + docker compose up -d webknossos + docker compose up -d webknossos-datastore + docker compose up -d webknossos-tracingstore - name: Run webknossos smoke test run: | @@ -174,7 +174,7 @@ jobs: (exit $s) - name: Stop webknossos, datastore, and tracingstore - run: docker-compose down --volumes --remove-orphans + run: docker compose down --volumes --remove-orphans - name: Push docker images run: | @@ -186,9 +186,9 @@ jobs: return $s } retry docker login -u $DOCKER_USER -p $DOCKER_PASS - retry docker-compose push webknossos - retry docker-compose push webknossos-datastore - retry docker-compose push webknossos-tracingstore + retry docker compose push webknossos + retry docker compose push webknossos-datastore + retry docker compose push webknossos-tracingstore if [[ ${{ github.ref_type }} == "branch" ]]; then docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} @@ -202,7 +202,7 @@ jobs: docker logout - name: Report coverage - run: .github/not-on-master.sh docker-compose run base yarn coverage || true + run: .github/not-on-master.sh docker compose run base yarn coverage || true # - name: Send Slack notification (master only) # run: .circleci/slack-notification.sh From d0eed33509de93c2e96cb5dfbf86c612a3053347 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Thu, 24 Oct 2024 16:09:27 +0200 Subject: [PATCH 03/43] only run github actions on PRs and master branch --- .github/workflows/build_test_deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 506cf1ad82..57e90dfd66 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -3,14 +3,14 @@ name: CI Pipeline on: push: branches: - - '*' + - master pull_request: branches: - '*' workflow_dispatch: env: - USER_NAME: circleci + USER_NAME: root USER_UID: 1000 USER_GID: 1000 TZ: Europe/Berlin @@ -18,7 +18,7 @@ env: DOCKER_PASS: ${{ secrets.DOCKER_PASS }} jobs: - static_frontent_code_checks: + static_frontend_code_checks: runs-on: ubuntu-20.04 steps: - name: Checkout code From 53e35c463c9ae9ed03eefd3f398afb29ddfd3893 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 08:31:45 +0100 Subject: [PATCH 04/43] CI --- .github/actions/health_check_action.yml | 16 ++++++++++++++ .github/workflows/build_test_deploy.yml | 29 +++++++++---------------- docker-compose.yml | 4 ++-- project/BuildInfoSettings.scala | 4 ++-- 4 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 .github/actions/health_check_action.yml diff --git a/.github/actions/health_check_action.yml b/.github/actions/health_check_action.yml new file mode 100644 index 0000000000..36d180d96e --- /dev/null +++ b/.github/actions/health_check_action.yml @@ -0,0 +1,16 @@ +name: 'URL Health Check' +description: 'Performs URL health check with retries' +inputs: + url: + description: 'Health check URL' + required: true +runs: + using: 'composite' + steps: + - shell: bash + run: | + for i in {1..20}; do + curl --fail -v "${{ inputs.url }}" && exit 0 + sleep 5 + done + exit 1 \ No newline at end of file diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 57e90dfd66..1e4a0614f8 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -13,7 +13,6 @@ env: USER_NAME: root USER_UID: 1000 USER_GID: 1000 - TZ: Europe/Berlin DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} @@ -150,28 +149,20 @@ jobs: docker compose up -d webknossos-tracingstore - name: Run webknossos smoke test - run: | - for i in {1..20}; do # retry - curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$? - sleep 5 - done - (exit $s) + uses: ./.github/actions/health_check_action + with: + url: http://localhost:9000/api/health + - name: Run webknossos-datastore smoke test - run: | - for i in {1..20}; do # retry - curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$? - sleep 5 - done - (exit $s) + uses: ./.github/actions/health_check_action + with: + url: http://localhost:9090/data/health - name: Run webknossos-tracingstore smoke test - run: | - for i in {1..20}; do # retry - curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$? - sleep 5 - done - (exit $s) + uses: ./.github/actions/health_check_action + with: + url: http://localhost:9050/tracings/health - name: Stop webknossos, datastore, and tracingstore run: docker compose down --volumes --remove-orphans diff --git a/docker-compose.yml b/docker-compose.yml index 76f802426b..70232a2356 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,8 +105,8 @@ services: - POSTGRES_USER=webknossos_user - POSTGRES_PASSWORD=secret_password - HOME=/root - - CIRCLE_TAG=${CIRCLE_TAG} - - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM} + - CI_TAG=${GITHUB_REF} + - CI_BUILD_NUM=${GITHUB_JOB} - NODE_OPTIONS=--max_old_space_size=2048 # This will be picked up by ava so that the tests fail if a snapshot doesn't exist during CI, see https://github.com/avajs/ava/issues/1585 - CI diff --git a/project/BuildInfoSettings.scala b/project/BuildInfoSettings.scala index a16d0223fe..229e55b575 100644 --- a/project/BuildInfoSettings.scala +++ b/project/BuildInfoSettings.scala @@ -13,8 +13,8 @@ object BuildInfoSettings { } val ciBuild: String = - if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else "" - val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else "" + if (System.getenv().containsKey("CI_BUILD_NUM")) System.getenv().get("CI_BUILD_NUM") else "" + val ciTag: String = if (System.getenv().containsKey("CI_TAG")) System.getenv().get("CI_TAG") else "" def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "") def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "") From 53d1c23a65ca7f3c2f78567bfdb3e33738edccc1 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 08:41:17 +0100 Subject: [PATCH 05/43] fix yarn lockfile --- yarn.lock | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/yarn.lock b/yarn.lock index f1575bb6d8..addd51b547 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8109,7 +8109,7 @@ __metadata: languageName: node linkType: hard -"lodash-es@npm:^4.17.21, lodash-es@npm:^4.2.1": +"lodash-es@npm:^4.17.21": version: 4.17.21 resolution: "lodash-es@npm:4.17.21" checksum: 10c0/fb407355f7e6cd523a9383e76e6b455321f0f153a6c9625e21a8827d10c54c2a2341bd2ae8d034358b60e07325e1330c14c224ff582d04612a46a4f0479ff2f2 @@ -8144,7 +8144,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21, lodash@npm:^4.2.1": +"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c @@ -11587,14 +11587,11 @@ __metadata: linkType: hard "redux@npm:^4.2.0": - version: 3.7.2 - resolution: "redux@npm:3.7.2" + version: 4.2.1 + resolution: "redux@npm:4.2.1" dependencies: - lodash: "npm:^4.2.1" - lodash-es: "npm:^4.2.1" - loose-envify: "npm:^1.1.0" - symbol-observable: "npm:^1.0.3" - checksum: 10c0/544456f95734de33326637b370894addb57d9de2524edf36a20e4a326d0a36a0e223979d027545c5aa8a8d7a2859363981f63d1146401b72df0d16f373dd09cb + "@babel/runtime": "npm:^7.9.2" + checksum: 10c0/136d98b3d5dbed1cd6279c8c18a6a74c416db98b8a432a46836bdd668475de6279a2d4fd9d1363f63904e00f0678a8a3e7fa532c897163340baf1e71bb42c742 languageName: node linkType: hard @@ -12835,13 +12832,6 @@ __metadata: languageName: node linkType: hard -"symbol-observable@npm:^1.0.3": - version: 1.2.0 - resolution: "symbol-observable@npm:1.2.0" - checksum: 10c0/009fee50798ef80ed4b8195048288f108b03de162db07493f2e1fd993b33fafa72d659e832b584da5a2427daa78e5a738fb2a9ab027ee9454252e0bedbcd1fdc - languageName: node - linkType: hard - "symbol-tree@npm:^3.2.4": version: 3.2.4 resolution: "symbol-tree@npm:3.2.4" From eb8af0bf2c7ed41d0571475a369c2a8c05f3d378 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 09:00:09 +0100 Subject: [PATCH 06/43] CI --- .github/workflows/build_test_deploy.yml | 4 ++-- docker-compose.yml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 1e4a0614f8..8afc349cd7 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: env: - USER_NAME: root + USER_NAME: webknossos USER_UID: 1000 USER_GID: 1000 DOCKER_USER: ${{ secrets.DOCKER_USER }} @@ -76,7 +76,7 @@ jobs: run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier - name: Install frontend dependencies - run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable + run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean - name: Assert unique evolution numbers run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers diff --git a/docker-compose.yml b/docker-compose.yml index 70232a2356..131b190dbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "2.2" - services: # Production webknossos: From 1d77e3c8048cb4ca7455196d80358f0c5d2f10df Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 09:07:37 +0100 Subject: [PATCH 07/43] CI --- .github/workflows/build_test_deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 8afc349cd7..b6e79a27a0 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -10,9 +10,9 @@ on: workflow_dispatch: env: - USER_NAME: webknossos - USER_UID: 1000 - USER_GID: 1000 + # USER_NAME: root + # USER_UID: 1000 + # USER_GID: 1000 DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} From 03c1cbfa47b83b3bc6d95dabc53cfb9c5dabb0b1 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 09:31:09 +0100 Subject: [PATCH 08/43] Ci --- .github/workflows/build_test_deploy.yml | 8 +++----- Dockerfile | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index b6e79a27a0..b99fa61d6e 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -10,9 +10,9 @@ on: workflow_dispatch: env: - # USER_NAME: root - # USER_UID: 1000 - # USER_GID: 1000 + USER_NAME: root + USER_UID: 1000 + USER_GID: 1000 DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} @@ -42,8 +42,6 @@ jobs: run: yarn check-cyclic-dependencies - - build_test_deploy: runs-on: ubuntu-20.04 steps: diff --git a/Dockerfile b/Dockerfile index 2a553862cc..d3e12dc63c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \ RUN mkdir -p /webknossos WORKDIR /webknossos +# Copy compiled Scala output from a previous build step, e.g. output of the Docker-dev image COPY target/universal/stage . RUN addgroup --system --gid 999 webknossos \ From 7acf3a953f5474cb0ffa54045c897185990d12c6 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 13:53:49 +0100 Subject: [PATCH 09/43] CI --- .github/workflows/build_test_deploy.yml | 92 +++++++++++++++---------- 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index b99fa61d6e..16ef68853f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 18 - name: Install frontend dependencies run: corepack enable && yarn install --immutable @@ -61,51 +61,68 @@ jobs: echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV fi - - - name: Set up Docker - uses: docker/setup-buildx-action@v2 - - - name: Build webknossos-dev docker image - run: | - docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true - DEV_CACHE=$NORMALIZED_BRANCH docker compose build base - - - name: Prepare dependency folders - run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + - uses: actions/setup-node@v4 + with: + node-version: 18 - name: Install frontend dependencies - run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean - - - name: Assert unique evolution numbers - run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers - - - name: Assert schema.sql and evolutions are equal - run: | - docker compose up -d postgres - sleep 3 - docker compose run compile tools/postgres/dbtool.js check-evolutions-schema - - - name: Build frontend documentation - run: | - WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} - docker compose run base yarn run docs --project-version $WK_VERSION - + run: corepack enable && yarn install --immutable + - name: Build webknossos (webpack) - run: docker compose run base yarn build + run: yarn build + + - name: Set up Java / SBT + uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + cache: 'sbt' + + - name: Install dependencies + run: apt-get update \ + && apt-get install -y \ + findutils \ + libdraco-dev \ + libblosc1 + + # build-essential \ + # cmake \ + # - name: Set up Docker + # uses: docker/setup-buildx-action@v2 + + # - name: Build webknossos-dev docker image + # run: | + # docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true + # DEV_CACHE=$NORMALIZED_BRANCH docker compose build base + + # - name: Prepare dependency folders + # run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + + # - name: Install frontend dependencies + # run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean + + # - name: Assert unique evolution numbers + # run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers + + # - name: Assert schema.sql and evolutions are equal + # run: | + # docker compose up -d postgres + # sleep 3 + # docker compose run compile tools/postgres/dbtool.js check-evolutions-schema - name: Build webknossos (sbt) run: | if [ "${{ github.ref }}" == "refs/heads/master" ]; then - docker compose run compile sbt -no-colors clean compile stage + sbt -no-colors clean compile stage else - docker compose run compile sbt -no-colors -DfailOnWarning compile stage + sbt -no-colors -DfailOnWarning compile stage fi - name: Build webknossos-datastore (sbt) - run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage - name: Build webknossos-tracingstore (sbt) - run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage - name: Checksum App Dirs run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt @@ -137,9 +154,6 @@ jobs: done (exit $s) - - name: Validate frontend types - run: .github/not-on-master.sh docker compose run base yarn typecheck - - name: Start webknossos, datastore, and tracingstore run: | docker compose up -d webknossos @@ -190,6 +204,12 @@ jobs: fi docker logout + - name: Build frontend documentation + run: | + WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} + yarn run docs --project-version $WK_VERSION + + - name: Report coverage run: .github/not-on-master.sh docker compose run base yarn coverage || true From 633b9bf20caea7538c575ac43b5a7b105211b8c5 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:03:54 +0100 Subject: [PATCH 10/43] CI --- .github/workflows/build_test_deploy.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 16ef68853f..af2e0a6cba 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -78,9 +78,8 @@ jobs: java-version: '21' cache: 'sbt' - - name: Install dependencies - run: apt-get update \ - && apt-get install -y \ + - name: Install OS dependencies / libs + run: apt-get update && apt-get install -y \ findutils \ libdraco-dev \ libblosc1 From 267d6468d7dd7979932f8de4087330f419624e47 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:06:16 +0100 Subject: [PATCH 11/43] sudo! --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index af2e0a6cba..90941d0e3a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -79,7 +79,7 @@ jobs: cache: 'sbt' - name: Install OS dependencies / libs - run: apt-get update && apt-get install -y \ + run: sudo apt-get update && apt-get install -y \ findutils \ libdraco-dev \ libblosc1 From 188753d532d4acbae4d9edd8fcf527616472bb54 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:08:03 +0100 Subject: [PATCH 12/43] sudo --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 90941d0e3a..3f38d6c38e 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -79,7 +79,7 @@ jobs: cache: 'sbt' - name: Install OS dependencies / libs - run: sudo apt-get update && apt-get install -y \ + run: sudo apt-get update && sudo apt-get install -y \ findutils \ libdraco-dev \ libblosc1 From 47e66d6adf1a01b21201ea275bf4da2087ed3e69 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:13:08 +0100 Subject: [PATCH 13/43] bump ubuntu version --- .github/workflows/build_test_deploy.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3f38d6c38e..5c13f4c59a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -18,7 +18,7 @@ env: jobs: static_frontend_code_checks: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v3 @@ -86,20 +86,10 @@ jobs: # build-essential \ # cmake \ - # - name: Set up Docker - # uses: docker/setup-buildx-action@v2 - - # - name: Build webknossos-dev docker image - # run: | - # docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true - # DEV_CACHE=$NORMALIZED_BRANCH docker compose build base # - name: Prepare dependency folders # run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier - # - name: Install frontend dependencies - # run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable && yarn cache clean - # - name: Assert unique evolution numbers # run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers @@ -126,6 +116,9 @@ jobs: - name: Checksum App Dirs run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + - name: Build webknossos docker image run: | docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true @@ -137,9 +130,6 @@ jobs: - name: Build webknossos-tracingstore docker image run: docker compose build --pull webknossos-tracingstore - - name: Run frontend tests - run: .github/not-on-master.sh docker compose run base yarn test-verbose - - name: Lint backend code and check formatting run: .github/not-on-master.sh docker compose run backend-lint-format @@ -164,7 +154,6 @@ jobs: with: url: http://localhost:9000/api/health - - name: Run webknossos-datastore smoke test uses: ./.github/actions/health_check_action with: From f69e1c9a5281f635e92aceadac37c1a85ef0d904 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:13:54 +0100 Subject: [PATCH 14/43] CI --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 5c13f4c59a..e7d13d4668 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -43,7 +43,7 @@ jobs: build_test_deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v3 From 98337a47fa1ea6925dcf6e179340bea2f8d2abcf Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:17:48 +0100 Subject: [PATCH 15/43] use different package names for ubuntu --- .github/workflows/build_test_deploy.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e7d13d4668..e9ff84b86a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -80,9 +80,8 @@ jobs: - name: Install OS dependencies / libs run: sudo apt-get update && sudo apt-get install -y \ - findutils \ - libdraco-dev \ - libblosc1 + draco \ + c-blosc # build-essential \ # cmake \ From da87a2a0b5df6b7cb769d6a15fcb3cda27e3b6c4 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:27:45 +0100 Subject: [PATCH 16/43] CI --- .github/workflows/build_test_deploy.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e9ff84b86a..d1a9acf4b7 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -44,6 +44,25 @@ jobs: build_test_deploy: runs-on: ubuntu-24.04 + env: + # The hostname used to communicate with the PostgreSQL service container + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_DB: webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 2s + --health-timeout 5s + --health-retries 30 + ports: + - 5432:5432 steps: - name: Checkout code uses: actions/checkout@v3 @@ -80,8 +99,8 @@ jobs: - name: Install OS dependencies / libs run: sudo apt-get update && sudo apt-get install -y \ - draco \ - c-blosc + libdraco-dev \ + libblosc-dev # build-essential \ # cmake \ From a15f95c9cc0f09710432bcbc282a489b053c3cb4 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:34:41 +0100 Subject: [PATCH 17/43] enable ssh debug --- .github/workflows/build_test_deploy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index d1a9acf4b7..9071aa8645 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -64,6 +64,10 @@ jobs: ports: - 5432:5432 steps: + # Enable tmate debugging of manually-triggered workflows if the input option was provided + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - name: Checkout code uses: actions/checkout@v3 with: From 5a2766fc1390f62898150c8b784ae8ced6842966 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:36:25 +0100 Subject: [PATCH 18/43] CI --- .github/workflows/build_test_deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 9071aa8645..8b59f73c99 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -8,6 +8,12 @@ on: branches: - '*' workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false env: USER_NAME: root From 35e7381300e90a4ec789a84a10605a55e2d843fc Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:40:54 +0100 Subject: [PATCH 19/43] CI --- .github/workflows/build_test_deploy.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 8b59f73c99..ebba6f6139 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -108,9 +108,7 @@ jobs: cache: 'sbt' - name: Install OS dependencies / libs - run: sudo apt-get update && sudo apt-get install -y \ - libdraco-dev \ - libblosc-dev + run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev # build-essential \ # cmake \ From e81a452fe2c4d21dc27c7f9253428c43af6bc76d Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:48:53 +0100 Subject: [PATCH 20/43] add sbt action --- .github/workflows/build_test_deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index ebba6f6139..567124714e 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -100,13 +100,15 @@ jobs: - name: Build webknossos (webpack) run: yarn build - - name: Set up Java / SBT + - name: Set up Java uses: actions/setup-java@v4 with: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '21' cache: 'sbt' - + - name: Set up sbt + uses: sbt/setup-sbt@v1 + - name: Install OS dependencies / libs run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev From e3bd80464a00fde82ce1768f878e14df36f5c8bc Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 14:54:36 +0100 Subject: [PATCH 21/43] ci --- .github/workflows/build_test_deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 567124714e..b045589435 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -59,7 +59,7 @@ jobs: image: postgres:10-alpine env: POSTGRES_DB: webknossos - POSTGRES_USER: webknossos_user + POSTGRES_USER: postgres POSTGRES_PASSWORD: secret_password # Set health checks to wait until postgres has started options: >- @@ -108,7 +108,7 @@ jobs: cache: 'sbt' - name: Set up sbt uses: sbt/setup-sbt@v1 - + - name: Install OS dependencies / libs run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc-dev From d12c11004f1a98bbffb15cb4749d417eed92fe60 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 28 Oct 2024 15:01:14 +0100 Subject: [PATCH 22/43] ci --- .github/workflows/build_test_deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index b045589435..5fa31e45bc 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -52,8 +52,9 @@ jobs: runs-on: ubuntu-24.04 env: # The hostname used to communicate with the PostgreSQL service container - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 + # POSTGRES_HOST: localhost + # POSTGRES_PORT: 5432 + PGPASSWORD: secret_password services: postgres: image: postgres:10-alpine From b91701de2f3e3b84f71c7f543b16466b3308ca74 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 30 Oct 2024 15:05:08 +0100 Subject: [PATCH 23/43] CI --- .github/workflows/build_test_deploy.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 5fa31e45bc..9deb2d2eb7 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -59,8 +59,8 @@ jobs: postgres: image: postgres:10-alpine env: - POSTGRES_DB: webknossos - POSTGRES_USER: postgres + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user POSTGRES_PASSWORD: secret_password # Set health checks to wait until postgres has started options: >- @@ -152,6 +152,12 @@ jobs: run: | docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos + env: + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + PGUSER: webknossos_user + POSTGRES_PASSWORD: secret_password + PGPASSWORD: secret_password - name: Build webknossos-datastore docker image run: docker compose build --pull webknossos-datastore From c15cc6be6e749c95f1846c9119351bca5a44b81f Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 30 Oct 2024 15:13:07 +0100 Subject: [PATCH 24/43] CI --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 131b190dbf..60978e4ad4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: - USER_GID - COVERALLS_REPO_TOKEN - TZ=${TZ:-Europe/Berlin} - - POSTGRES_URL=jdbc:postgresql://postgres/webknossos + - POSTGRES_URL=jdbc:postgresql://postgres@localhost:5432/webknossos - POSTGRES_USER=webknossos_user - POSTGRES_PASSWORD=secret_password - HOME=/root From e3c6b4d84a8151676a944cdd0a44f94ab6f9b1c4 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 30 Oct 2024 15:20:10 +0100 Subject: [PATCH 25/43] CI --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 60978e4ad4..96bb979361 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: - USER_GID - COVERALLS_REPO_TOKEN - TZ=${TZ:-Europe/Berlin} - - POSTGRES_URL=jdbc:postgresql://postgres@localhost:5432/webknossos + - POSTGRES_URL=jdbc:postgresql://webknossos_user@localhost:5432/webknossos - POSTGRES_USER=webknossos_user - POSTGRES_PASSWORD=secret_password - HOME=/root From 543bba102e518c5218cde7f96cc5abdddebe6757 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 30 Oct 2024 15:21:23 +0100 Subject: [PATCH 26/43] CI --- .github/workflows/build_test_deploy.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 9deb2d2eb7..ccd54642fe 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -153,7 +153,7 @@ jobs: docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos env: - POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_URL: jdbc:postgresql://webknossos_user@localhost:5432/webknossos POSTGRES_USER: webknossos_user PGUSER: webknossos_user POSTGRES_PASSWORD: secret_password diff --git a/docker-compose.yml b/docker-compose.yml index 96bb979361..131b190dbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: - USER_GID - COVERALLS_REPO_TOKEN - TZ=${TZ:-Europe/Berlin} - - POSTGRES_URL=jdbc:postgresql://webknossos_user@localhost:5432/webknossos + - POSTGRES_URL=jdbc:postgresql://postgres/webknossos - POSTGRES_USER=webknossos_user - POSTGRES_PASSWORD=secret_password - HOME=/root From f25cca044d8951cb5ec8d3cd8b6c152f33a7d5da Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 15:38:03 +0100 Subject: [PATCH 27/43] enable ssh debugging --- .github/workflows/build_test_deploy.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index ccd54642fe..c7bcaa4ba6 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -72,9 +72,7 @@ jobs: - 5432:5432 steps: # Enable tmate debugging of manually-triggered workflows if the input option was provided - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + - name: Checkout code uses: actions/checkout@v3 with: @@ -127,6 +125,9 @@ jobs: # docker compose up -d postgres # sleep 3 # docker compose run compile tools/postgres/dbtool.js check-evolutions-schema + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - name: Build webknossos (sbt) run: | From e6de5cb1e79dbb7189c489e83775d70a645294d3 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 16:00:38 +0100 Subject: [PATCH 28/43] set postgres url --- .github/workflows/build_test_deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index c7bcaa4ba6..92824a8d97 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -118,7 +118,7 @@ jobs: # run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier # - name: Assert unique evolution numbers - # run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers + # run: node tools/postgres/dbtool.js assert-unique-evolution-numbers # - name: Assert schema.sql and evolutions are equal # run: | @@ -136,6 +136,9 @@ jobs: else sbt -no-colors -DfailOnWarning compile stage fi + env: + POSTGRES_URL: postgres://webknossos_user:secret_password@localhost:5432/webknossos + - name: Build webknossos-datastore (sbt) run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage From 816989dfa8911a86d6e0113979d6186373a5179e Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 16:05:28 +0100 Subject: [PATCH 29/43] CI --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 92824a8d97..29128f4bf0 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -137,7 +137,7 @@ jobs: sbt -no-colors -DfailOnWarning compile stage fi env: - POSTGRES_URL: postgres://webknossos_user:secret_password@localhost:5432/webknossos + POSTGRES_URL: jdbc:postgres://webknossos_user:secret_password@localhost:5432/webknossos - name: Build webknossos-datastore (sbt) From 495296ea3001ee475d9adc5a36f5c64edc76758b Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 16:22:53 +0100 Subject: [PATCH 30/43] CI --- .github/workflows/build_test_deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 29128f4bf0..d2109c7f52 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -137,8 +137,7 @@ jobs: sbt -no-colors -DfailOnWarning compile stage fi env: - POSTGRES_URL: jdbc:postgres://webknossos_user:secret_password@localhost:5432/webknossos - + POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos - name: Build webknossos-datastore (sbt) run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage From d9322ce5946f69c6abf3444195120b64ecbb1b29 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 16:42:02 +0100 Subject: [PATCH 31/43] ci --- .github/workflows/build_test_deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index d2109c7f52..639759e29f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -138,6 +138,8 @@ jobs: fi env: POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password - name: Build webknossos-datastore (sbt) run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage From 035f5a3a532e23592f75a773aa62c5c7eb97a699 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 16:59:35 +0100 Subject: [PATCH 32/43] ci --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 639759e29f..73523aa945 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -137,7 +137,7 @@ jobs: sbt -no-colors -DfailOnWarning compile stage fi env: - POSTGRES_URL: jdbc:postgresql://webknossos_user:secret_password@localhost:5432/webknossos + POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos POSTGRES_USER: webknossos_user POSTGRES_PASSWORD: secret_password From b162be248cb1d5be7abbabf72cf2dc48d29a6e10 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 17:18:29 +0100 Subject: [PATCH 33/43] ci --- .github/workflows/build_test_deploy.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 73523aa945..5176540efa 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -71,7 +71,6 @@ jobs: ports: - 5432:5432 steps: - # Enable tmate debugging of manually-triggered workflows if the input option was provided - name: Checkout code uses: actions/checkout@v3 @@ -125,10 +124,8 @@ jobs: # docker compose up -d postgres # sleep 3 # docker compose run compile tools/postgres/dbtool.js check-evolutions-schema - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - + + - name: Build webknossos (sbt) run: | if [ "${{ github.ref }}" == "refs/heads/master" ]; then @@ -158,11 +155,9 @@ jobs: docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos env: - POSTGRES_URL: jdbc:postgresql://webknossos_user@localhost:5432/webknossos + POSTGRES_URL: jdbc:postgresql://localhost:5432/webknossos POSTGRES_USER: webknossos_user - PGUSER: webknossos_user POSTGRES_PASSWORD: secret_password - PGPASSWORD: secret_password - name: Build webknossos-datastore docker image run: docker compose build --pull webknossos-datastore @@ -170,6 +165,11 @@ jobs: - name: Build webknossos-tracingstore docker image run: docker compose build --pull webknossos-tracingstore + # Enable tmate debugging of manually-triggered workflows if the input option was provided + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + - name: Lint backend code and check formatting run: .github/not-on-master.sh docker compose run backend-lint-format From 6c06ca31ffd0a785c390b50128cf4ed62aad888b Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 17:27:27 +0100 Subject: [PATCH 34/43] CI --- .github/not-on-master.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/not-on-master.sh diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh old mode 100644 new mode 100755 From 18b7cb42f482bd66c0e24d029d1e9d1a183c63f9 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Mon, 11 Nov 2024 17:29:23 +0100 Subject: [PATCH 35/43] ci --- .github/not-on-master.sh | 2 +- .github/workflows/build_test_deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/not-on-master.sh b/.github/not-on-master.sh index fc9ccc8fc0..b2bd8536de 100755 --- a/.github/not-on-master.sh +++ b/.github/not-on-master.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -if [ "${GITHUB_REF}" == "master" ]; then +if [ "${GITHUB_REF_NAME}" == "master" ]; then echo "Skipping this step on master..." else exec "$@" diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 5176540efa..b7be8e1923 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -128,7 +128,7 @@ jobs: - name: Build webknossos (sbt) run: | - if [ "${{ github.ref }}" == "refs/heads/master" ]; then + if [ "${{ github.ref_name }}" == "master" ]; then sbt -no-colors clean compile stage else sbt -no-colors -DfailOnWarning compile stage From 77d54be8b55f49e59abcd3a7f7fa39cf66906bcf Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Tue, 12 Nov 2024 11:58:44 +0100 Subject: [PATCH 36/43] ci --- .github/workflows/build_test_deploy.yml | 66 +++++++++++-------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index b7be8e1923..1ae1770440 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -144,6 +144,9 @@ jobs: - name: Build webknossos-tracingstore (sbt) run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + - name: Lint backend code and check formatting + run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + - name: Checksum App Dirs run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt @@ -170,18 +173,16 @@ jobs: uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Lint backend code and check formatting - run: .github/not-on-master.sh docker compose run backend-lint-format - name: Run backend tests run: .github/not-on-master.sh docker compose run backend-tests - name: Run end-to-end tests - run: | - for i in {1..3}; do # retry - .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$? - done - (exit $s) + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: .github/not-on-master.sh docker compose run e2e-tests - name: Start webknossos, datastore, and tracingstore run: | @@ -208,38 +209,29 @@ jobs: run: docker compose down --volumes --remove-orphans - name: Push docker images - run: | - function retry() { - for i in {1..5}; do - "$@" && s=0 && break || s=$? - sleep 10 - done - return $s - } - retry docker login -u $DOCKER_USER -p $DOCKER_PASS - retry docker compose push webknossos - retry docker compose push webknossos-datastore - retry docker compose push webknossos-tracingstore - if [[ ${{ github.ref_type }} == "branch" ]]; then - docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} - retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} - docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} - retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} - docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} - retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} - docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH} - retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH} - fi - docker logout - - - name: Build frontend documentation - run: | - WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }} - yarn run docs --project-version $WK_VERSION - + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + docker compose push webknossos + docker compose push webknossos-datastore + docker compose push webknossos-tracingstore + + if [[ ${{ github.ref_type }} == "branch" ]]; then + docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH} + docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH} + fi + docker logout - name: Report coverage - run: .github/not-on-master.sh docker compose run base yarn coverage || true + run: .github/not-on-master.sh yarn coverage || true # - name: Send Slack notification (master only) # run: .circleci/slack-notification.sh From 46f66962503c0efeac609335bf3d6d25cc7a0caf Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Tue, 12 Nov 2024 13:35:55 +0100 Subject: [PATCH 37/43] ci --- .github/workflows/build_test_deploy.yml | 23 +++-- docker-compose.yml | 126 ++++++++++++------------ 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 1ae1770440..b779699eb8 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -147,6 +147,9 @@ jobs: - name: Lint backend code and check formatting run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + - name: Run backend tests + run: .github/not-on-master.sh sbt -v "testOnly backend.*" + - name: Checksum App Dirs run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt @@ -172,18 +175,7 @@ jobs: - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - - - name: Run backend tests - run: .github/not-on-master.sh docker compose run backend-tests - - - name: Run end-to-end tests - uses: nick-invision/retry@v2 - with: - timeout_minutes: 10 - max_attempts: 3 - command: .github/not-on-master.sh docker compose run e2e-tests - + - name: Start webknossos, datastore, and tracingstore run: | docker compose up -d webknossos @@ -204,6 +196,13 @@ jobs: uses: ./.github/actions/health_check_action with: url: http://localhost:9050/tracings/health + + - name: Run end-to-end tests + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + command: .github/not-on-master.sh yarn run test-e2e - name: Stop webknossos, datastore, and tracingstore run: docker compose down --volumes --remove-orphans diff --git a/docker-compose.yml b/docker-compose.yml index 131b190dbf..00d39873b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,62 +116,62 @@ services: - "${DOCKER_CACHE_PREFIX:-~}/.cache/yarn:/root/.cache/yarn" user: ${USER_UID:-1000}:${USER_GID:-1000} - compile: - extends: - service: base - depends_on: - postgres: - condition: service_healthy + # compile: + # extends: + # service: base + # depends_on: + # postgres: + # condition: service_healthy - dev: - extends: - service: base - ports: - - "5005:5005" - - "9000:9000" - links: - - "fossildb-dev:fossildb" - - "postgres-dev:postgres" - depends_on: - redis: - condition: service_healthy - postgres-dev: - condition: service_healthy - fossildb-dev: - condition: service_healthy - environment: - POSTGRES_URL: jdbc:postgresql://postgres/webknossos - POSTGRES_USER: webknossos_user - POSTGRES_PASSWORD: secret_password - command: - - bash - - -c - - > - sbt - -v -d -jvm-debug 5005 - "run - -Djava.net.preferIPv4Stack=true - -Dtracingstore.fossildb.address=fossildb - -Dtracingstore.redis.address=redis" - -Ddatastore.redis.address=redis" - stdin_open: true + # dev: + # extends: + # service: base + # ports: + # - "5005:5005" + # - "9000:9000" + # links: + # - "fossildb-dev:fossildb" + # - "postgres-dev:postgres" + # depends_on: + # redis: + # condition: service_healthy + # postgres-dev: + # condition: service_healthy + # fossildb-dev: + # condition: service_healthy + # environment: + # POSTGRES_URL: jdbc:postgresql://postgres/webknossos + # POSTGRES_USER: webknossos_user + # POSTGRES_PASSWORD: secret_password + # command: + # - bash + # - -c + # - > + # sbt + # -v -d -jvm-debug 5005 + # "run + # -Djava.net.preferIPv4Stack=true + # -Dtracingstore.fossildb.address=fossildb + # -Dtracingstore.redis.address=redis" + # -Ddatastore.redis.address=redis" + # stdin_open: true - # Tests - backend-tests: - extends: - service: base - ports: - - "5005:5005" - - "9000:9000" - command: sbt -v "testOnly backend.*" + # # Tests + # backend-tests: + # extends: + # service: base + # ports: + # - "5005:5005" + # - "9000:9000" + # command: sbt -v "testOnly backend.*" - backend-lint-format: - extends: - service: base - ports: - - "5005:5005" - - "9000:9000" - command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + # backend-lint-format: + # extends: + # service: base + # ports: + # - "5005:5005" + # - "9000:9000" + # command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" e2e-tests: extends: @@ -201,16 +201,16 @@ services: volumes: - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X - screenshot-tests: - image: scalableminds/puppeteer:master - environment: - - URL - - WK_AUTH_TOKEN - working_dir: /home/pptruser/webknossos - command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done' - volumes: - - ".:/home/pptruser/webknossos" - user: ${USER_UID:-1000}:${USER_GID:-1000} + # screenshot-tests: + # image: scalableminds/puppeteer:master + # environment: + # - URL + # - WK_AUTH_TOKEN + # working_dir: /home/pptruser/webknossos + # command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done' + # volumes: + # - ".:/home/pptruser/webknossos" + # user: ${USER_UID:-1000}:${USER_GID:-1000} # Postgres postgres: From e864dbb2d72cab2b6c03fd0e6c0ff8a3af6c2ba2 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Tue, 12 Nov 2024 13:44:42 +0100 Subject: [PATCH 38/43] ci --- docker-compose.yml | 54 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 00d39873b0..2c8f336700 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,33 +173,33 @@ services: # - "9000:9000" # command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" - e2e-tests: - extends: - service: backend-tests - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - fossildb: - condition: service_healthy - environment: - POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing - POSTGRES_USER: webknossos_user - POSTGRES_PASSWORD: secret_password - command: - - bash - - -c - - > - sbt - -v - "testOnly e2e.* -- - -Dtracingstore.fossildb.address=fossildb - -Dtracingstore.redis.address=redis - -Ddatastore.redis.address=redis - -Ddatastore.watchFileSystem.enabled=false" - volumes: - - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X + # e2e-tests: + # extends: + # service: backend-tests + # depends_on: + # redis: + # condition: service_healthy + # postgres: + # condition: service_healthy + # fossildb: + # condition: service_healthy + # environment: + # POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing + # POSTGRES_USER: webknossos_user + # POSTGRES_PASSWORD: secret_password + # command: + # - bash + # - -c + # - > + # sbt + # -v + # "testOnly e2e.* -- + # -Dtracingstore.fossildb.address=fossildb + # -Dtracingstore.redis.address=redis + # -Ddatastore.redis.address=redis + # -Ddatastore.watchFileSystem.enabled=false" + # volumes: + # - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X # screenshot-tests: # image: scalableminds/puppeteer:master From c9f8868a209c47e906e867934213dfbd35839068 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Tue, 12 Nov 2024 14:07:31 +0100 Subject: [PATCH 39/43] ci --- docker-compose.yml | 180 ++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2c8f336700..131b190dbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,101 +116,101 @@ services: - "${DOCKER_CACHE_PREFIX:-~}/.cache/yarn:/root/.cache/yarn" user: ${USER_UID:-1000}:${USER_GID:-1000} - # compile: - # extends: - # service: base - # depends_on: - # postgres: - # condition: service_healthy + compile: + extends: + service: base + depends_on: + postgres: + condition: service_healthy - # dev: - # extends: - # service: base - # ports: - # - "5005:5005" - # - "9000:9000" - # links: - # - "fossildb-dev:fossildb" - # - "postgres-dev:postgres" - # depends_on: - # redis: - # condition: service_healthy - # postgres-dev: - # condition: service_healthy - # fossildb-dev: - # condition: service_healthy - # environment: - # POSTGRES_URL: jdbc:postgresql://postgres/webknossos - # POSTGRES_USER: webknossos_user - # POSTGRES_PASSWORD: secret_password - # command: - # - bash - # - -c - # - > - # sbt - # -v -d -jvm-debug 5005 - # "run - # -Djava.net.preferIPv4Stack=true - # -Dtracingstore.fossildb.address=fossildb - # -Dtracingstore.redis.address=redis" - # -Ddatastore.redis.address=redis" - # stdin_open: true + dev: + extends: + service: base + ports: + - "5005:5005" + - "9000:9000" + links: + - "fossildb-dev:fossildb" + - "postgres-dev:postgres" + depends_on: + redis: + condition: service_healthy + postgres-dev: + condition: service_healthy + fossildb-dev: + condition: service_healthy + environment: + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + command: + - bash + - -c + - > + sbt + -v -d -jvm-debug 5005 + "run + -Djava.net.preferIPv4Stack=true + -Dtracingstore.fossildb.address=fossildb + -Dtracingstore.redis.address=redis" + -Ddatastore.redis.address=redis" + stdin_open: true - # # Tests - # backend-tests: - # extends: - # service: base - # ports: - # - "5005:5005" - # - "9000:9000" - # command: sbt -v "testOnly backend.*" + # Tests + backend-tests: + extends: + service: base + ports: + - "5005:5005" + - "9000:9000" + command: sbt -v "testOnly backend.*" - # backend-lint-format: - # extends: - # service: base - # ports: - # - "5005:5005" - # - "9000:9000" - # command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + backend-lint-format: + extends: + service: base + ports: + - "5005:5005" + - "9000:9000" + command: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" - # e2e-tests: - # extends: - # service: backend-tests - # depends_on: - # redis: - # condition: service_healthy - # postgres: - # condition: service_healthy - # fossildb: - # condition: service_healthy - # environment: - # POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing - # POSTGRES_USER: webknossos_user - # POSTGRES_PASSWORD: secret_password - # command: - # - bash - # - -c - # - > - # sbt - # -v - # "testOnly e2e.* -- - # -Dtracingstore.fossildb.address=fossildb - # -Dtracingstore.redis.address=redis - # -Ddatastore.redis.address=redis - # -Ddatastore.watchFileSystem.enabled=false" - # volumes: - # - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X + e2e-tests: + extends: + service: backend-tests + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + fossildb: + condition: service_healthy + environment: + POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password + command: + - bash + - -c + - > + sbt + -v + "testOnly e2e.* -- + -Dtracingstore.fossildb.address=fossildb + -Dtracingstore.redis.address=redis + -Ddatastore.redis.address=redis + -Ddatastore.watchFileSystem.enabled=false" + volumes: + - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X - # screenshot-tests: - # image: scalableminds/puppeteer:master - # environment: - # - URL - # - WK_AUTH_TOKEN - # working_dir: /home/pptruser/webknossos - # command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done' - # volumes: - # - ".:/home/pptruser/webknossos" - # user: ${USER_UID:-1000}:${USER_GID:-1000} + screenshot-tests: + image: scalableminds/puppeteer:master + environment: + - URL + - WK_AUTH_TOKEN + working_dir: /home/pptruser/webknossos + command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done' + volumes: + - ".:/home/pptruser/webknossos" + user: ${USER_UID:-1000}:${USER_GID:-1000} # Postgres postgres: From 0be3a3baf951d0b95cd17f5b1fe368a71e4a7bea Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 4 Dec 2024 09:49:00 +0100 Subject: [PATCH 40/43] Update .github/workflows/build_test_deploy.yml Co-authored-by: robert-oleynik <62473688+robert-oleynik@users.noreply.github.com> --- .github/workflows/build_test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index b779699eb8..4d4ebfedae 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -178,6 +178,7 @@ jobs: - name: Start webknossos, datastore, and tracingstore run: | + mkdir fossildb && chmod 999:999 fossildb docker compose up -d webknossos docker compose up -d webknossos-datastore docker compose up -d webknossos-tracingstore From afb35b67328e3e4fa33d9cf3c87aeaf23ff8aaf9 Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 4 Dec 2024 09:52:54 +0100 Subject: [PATCH 41/43] Update .github/workflows/build_test_deploy.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 4d4ebfedae..2bcd5c14cc 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -178,7 +178,7 @@ jobs: - name: Start webknossos, datastore, and tracingstore run: | - mkdir fossildb && chmod 999:999 fossildb + mkdir fossildb && chown 999:999 fossildb docker compose up -d webknossos docker compose up -d webknossos-datastore docker compose up -d webknossos-tracingstore From d22dadeb1279127c3e9c0b874df70b74d96563ce Mon Sep 17 00:00:00 2001 From: Tom Herold Date: Wed, 4 Dec 2024 10:21:21 +0100 Subject: [PATCH 42/43] Update .github/workflows/build_test_deploy.yml Co-authored-by: robert-oleynik <62473688+robert-oleynik@users.noreply.github.com> --- .github/workflows/build_test_deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2bcd5c14cc..4f2c137176 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -178,7 +178,8 @@ jobs: - name: Start webknossos, datastore, and tracingstore run: | - mkdir fossildb && chown 999:999 fossildb + mkdir -p fossildb/data && chown -R 999:999 fossildb/data + mkdir -p fossildb/backup && chown -R 999:999 fossildb/backup docker compose up -d webknossos docker compose up -d webknossos-datastore docker compose up -d webknossos-tracingstore From c0c3db57a0453e1e9e56ed451049517ae366efe1 Mon Sep 17 00:00:00 2001 From: Robert Oleynik Date: Wed, 4 Dec 2024 16:56:42 +0100 Subject: [PATCH 43/43] fix directory permissions --- .github/workflows/build_test_deploy.yml | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 4f2c137176..d1c7f905d7 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -71,7 +71,7 @@ jobs: ports: - 5432:5432 steps: - + - name: Checkout code uses: actions/checkout@v3 with: @@ -94,7 +94,7 @@ jobs: - name: Install frontend dependencies run: corepack enable && yarn install --immutable - + - name: Build webknossos (webpack) run: yarn build @@ -124,8 +124,8 @@ jobs: # docker compose up -d postgres # sleep 3 # docker compose run compile tools/postgres/dbtool.js check-evolutions-schema - - + + - name: Build webknossos (sbt) run: | if [ "${{ github.ref_name }}" == "master" ]; then @@ -146,7 +146,7 @@ jobs: - name: Lint backend code and check formatting run: .github/not-on-master.sh sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" - + - name: Run backend tests run: .github/not-on-master.sh sbt -v "testOnly backend.*" @@ -175,30 +175,30 @@ jobs: - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - + - name: Start webknossos, datastore, and tracingstore run: | - mkdir -p fossildb/data && chown -R 999:999 fossildb/data - mkdir -p fossildb/backup && chown -R 999:999 fossildb/backup + mkdir -p fossildb/data && sudo chmod -R "0777" fossildb/data + mkdir -p fossildb/backup && sudo chmod -R "0777" fossildb/backup docker compose up -d webknossos docker compose up -d webknossos-datastore docker compose up -d webknossos-tracingstore - name: Run webknossos smoke test uses: ./.github/actions/health_check_action - with: + with: url: http://localhost:9000/api/health - + - name: Run webknossos-datastore smoke test uses: ./.github/actions/health_check_action - with: + with: url: http://localhost:9090/data/health - name: Run webknossos-tracingstore smoke test uses: ./.github/actions/health_check_action - with: + with: url: http://localhost:9050/tracings/health - + - name: Run end-to-end tests uses: nick-invision/retry@v2 with: @@ -218,7 +218,7 @@ jobs: docker compose push webknossos docker compose push webknossos-datastore docker compose push webknossos-tracingstore - + if [[ ${{ github.ref_type }} == "branch" ]]; then docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH} retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}