Merge pull request #86 from bacpop/cache-pipeline

.github/workflows/jestCI.yml

@@ -39,6 +39,8 @@ jobs:
 
   unit_and_integration_backend:
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     defaults:
       run:
         working-directory: ./app/server
@@ -66,10 +68,16 @@ jobs:
         run: |
           pwd
           cp ./src/resources/config.json.in.development ./src/resources/config.json
+      - name: Restore storage cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
+          restore-keys: |
+            dbs-cache-
       - name: run all components
         working-directory: .
-        run: ./scripts/run_test server-only
-
+        run: ./scripts/run_test server-only -mount ${{ env.STORAGE_PATH }}
       - name: Run unit and integration tests
         run: npm run test
       - name: stop all components

.github/workflows/playwrightCI.yml

@@ -13,6 +13,8 @@ jobs:
   playwright-tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    env:
+      STORAGE_PATH: ${{ github.workspace }}/storage/dbs
     strategy:
       fail-fast: false
       matrix:
@@ -26,6 +28,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Restore storage cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORAGE_PATH }}
+          key: dbs-cache-${{ runner.os }}
+          restore-keys: |
+            dbs-cache-
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -38,7 +47,7 @@ jobs:
           cp ./app/server/src/resources/config.json.in.development ./app/server/src/resources/config.json
       - name: Run all components
         working-directory: .
-        run: ./scripts/run_test
+        run: ./scripts/run_test -mount ${{ env.STORAGE_PATH }}
       - name: Install playwright
         working-directory: ./app/client-v2
         run: npx playwright install --with-deps

README.md

@@ -77,6 +77,8 @@ You can also run everything outside pm2, by separately running:
 - `./scripts/run_server`
 - `./scripts/run_client`
 
+*Note: If you wish to override the storage volume with a custom bind mount pass in -mount {MOUNT_NAME} into `run_test` script*
+
 ## Config
 Config for the front-end lives in `./app/client/src/settings` and by default webpack (via the vue-cli) will use the config 
 defined in `./app/client/src/settings/development`; this gets overriden by setting an env var called `BUILD_TARGET` - see `./proxy/Dockerfile`.

scripts/common

@@ -1,2 +1,9 @@
 #!/usr/bin/env bash
 export API_IMAGE="main"
+
+NETWORK=beebop_nw
+VOLUME=beebop-storage
+NAME_REDIS=beebop-redis
+NAME_API=beebop-py-api
+NAME_WORKER=beebop-py-worker
+PORT=5000

scripts/parse_mount_arg

@@ -0,0 +1,17 @@
+parse_mount_arg() {
+    local mount=""
+
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            -mount)
+                mount="$2"
+                shift 2
+                ;;
+            *)
+                shift
+                ;;
+        esac
+    done
+
+    echo "$mount"
+}

scripts/run_dependencies

@@ -4,32 +4,35 @@ set -ex
 HERE=$(realpath "$(dirname $0)")
 . $HERE/common
 
-NETWORK=beebop_nw
-VOLUME=beebop-storage
-NAME_REDIS=beebop-redis
-NAME_API=beebop-py-api
-NAME_WORKER=beebop-py-worker
-PORT=5000
-
-docker volume create $VOLUME
-docker run --rm -v $VOLUME:/beebop/storage \
+
+. $HERE/parse_mount_arg
+MOUNT=$(parse_mount_arg "$@")
+
+if [ -z "$MOUNT" ]; then
+       echo "No mount path provided, using default: $VOLUME"
+       MOUNT=$VOLUME
+       docker volume create $VOLUME
+fi
+
+docker network create $NETWORK > /dev/null || /bin/true
+
+docker run --rm -v $MOUNT:/beebop/storage \
        --pull always \
        ghcr.io/bacpop/beebop-py:$API_IMAGE \
        ./scripts/download_databases --refs # remove --refs to download all databases
-docker network create $NETWORK > /dev/null || /bin/true
 
 docker run -d --rm --name $NAME_REDIS --network=$NETWORK -p 6379:6379 redis:5.0
 docker run -d --rm --name $NAME_WORKER --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
-       -v $VOLUME:/beebop/storage \
+       -v $MOUNT:/beebop/storage \
        ghcr.io/bacpop/beebop-py:$API_IMAGE rqworker
 
 docker run -d --rm --name $NAME_API --network=$NETWORK \
        --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
        --env=STORAGE_LOCATION="./storage" \
        --env=DBS_LOCATION="./storage/dbs" \
-       -v $VOLUME:/beebop/storage \
+       -v $MOUNT:/beebop/storage \
        -p $PORT:5000 \
        ghcr.io/bacpop/beebop-py:$API_IMAGE

scripts/run_test

@@ -1,7 +1,11 @@
 set -e
 
 HERE=$(realpath "$(dirname $0)")
-. $HERE/run_dependencies
+
+. $HERE/parse_mount_arg
+MOUNT=$(parse_mount_arg "$@")
+. $HERE/run_dependencies -mount $MOUNT
+
 
 
 npm --prefix app/server ci