Merge pull request #8 from ebmdatalab/madwort/grafana-deployment

feat: add grafana deployment details

.github/dependabot.yml

@@ -11,3 +11,8 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
+
+  - package-ecosystem: "docker"
+    directory: "grafana/"
+    schedule:
+      interval: "weekly"

.github/workflows/grafana.yml

@@ -0,0 +1,112 @@
+---
+name: Grafana deployment
+
+env:
+  IMAGE_NAME: grafana
+  PUBLIC_IMAGE_NAME: ghcr.io/ebmdatalab/grafana
+  REGISTRY: ghcr.io
+  SSH_AUTH_SOCK: /tmp/agent.sock
+
+on:
+  push:
+
+jobs:
+  lint-dockerfile:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf  # v3.1.0
+        with:
+          dockerfile: grafana/Dockerfile
+
+  docker-test-and-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "opensafely-core/setup-action@v1"
+        with:
+          install-just: true
+
+      - name: Build docker image
+        run: |
+            just grafana/build 
+
+      - name: Run smoke test
+        run: |
+          just grafana/serve
+          sleep 5
+          just grafana/smoke-test || { docker logs grafana-grafana_1; exit 1; }
+
+      - name: Save docker image
+        run: |
+          docker save grafana | gzip > /tmp/grafana.tar.gz
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@v3
+        with:
+            name: grafana-image
+            path: /tmp/grafana.tar.gz
+
+  required-checks:
+    if: always()
+
+    needs:
+    - docker-test-and-build
+    - lint-dockerfile
+
+    runs-on: Ubuntu-latest
+
+    steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe  # v1.2.2
+      with:
+        jobs: ${{ toJSON(needs) }}
+
+  deploy:
+    needs: [required-checks]
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    if: github.ref == 'refs/heads/main'
+
+    concurrency: deploy-production
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "opensafely-core/setup-action@v1"
+        with:
+          install-just: true
+
+      - name: Download docker image
+        uses: actions/download-artifact@v3
+        with:
+            name: grafana-image
+            path: /tmp/image
+
+      - name: Import docker image
+        run: gunzip -c /tmp/image/grafana.tar.gz | docker load
+
+      - name: Test image we imported from previous job works
+        run: |
+            SKIP_BUILD=1 just grafana/serve
+            sleep 5
+            just grafana/smoke-test || { docker logs grafana-grafana_1; exit 1; }
+
+      - name: Publish image
+        run: |
+            echo ${{ secrets.GITHUB_TOKEN }} | docker login "$REGISTRY" -u ${{ github.actor }} --password-stdin
+            docker tag "$IMAGE_NAME" "$PUBLIC_IMAGE_NAME":latest
+            docker push "$PUBLIC_IMAGE_NAME":latest
+
+      - name: Deploy image
+        run: |
+            ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null
+            ssh-add - <<< "${{ secrets.DOKKU3_DEPLOY_SSH_KEY }}"
+            SHA=$(docker inspect --format='{{index .RepoDigests 0}}' "$PUBLIC_IMAGE_NAME":latest)
+            ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" [email protected] git:from-image grafana "$SHA"

docker-compose.yaml

@@ -11,7 +11,9 @@ services:
       - postgres:/var/lib/postgresql/data/
 
   grafana:
-    image: "grafana/grafana:latest"
+    # use Dockerfile so that version matches production
+    build:
+        dockerfile: grafana/Dockerfile
     environment:
       GF_DATABASE_TYPE: postgres
       GF_DATABASE_HOST: db:5432

grafana/Dockerfile

@@ -0,0 +1,3 @@
+FROM grafana/grafana-enterprise:latest@sha256:0d33bf1b2c3d508c5345ca652a96676c119420c4474e90299e4703ee834ade70
+
+USER grafana

grafana/README.md

@@ -0,0 +1,84 @@
+# Grafana deployment instructions
+## Create app
+
+```bash
+dokku$ dokku apps:create grafana
+dokku$ dokku domains:add grafana dashboards.opensafely.org
+```
+
+## Create postgresql db for grafana
+
+* on DO db cluster
+  * postgresql version to match target server - currently 14 on dokku3
+
+## create persistent storage
+
+```bash
+# the grafana container runs as uid 472 (grafana)
+# the other dokku containers on dokku3 run as uid 1013 (dokku)
+# let's tell the container to run as 1013, then we can use the same file permissions 
+dokku$ dokku docker-options:add grafana deploy "--user 1013"
+dokku$ dokku docker-options:add grafana run "--user 1013"
+dokku$ dokku storage:ensure-directory grafana
+
+myuser$ sudo chown -R dokku:dokku /var/lib/dokku/data/storage/grafana
+
+dokku$ dokku storage:mount grafana /var/lib/dokku/data/storage/grafana:/var/lib/grafana
+```
+
+## Configure app
+
+```bash
+dokku config:set grafana GF_DATABASE_TYPE="postgres"
+dokku config:set grafana GF_DATABASE_HOST="xxx:5432"
+dokku config:set grafana GF_DATABASE_NAME="grafana"
+dokku config:set grafana GF_DATABASE_USER="grafana"
+dokku config:set grafana GF_DATABASE_PASSWORD="xxx"
+dokku config:set grafana GF_DATABASE_SSL_MODE="require"
+dokku config:set grafana GF_SERVER_ROOT_URL="https://dashboards.opensafely.org/"
+dokku config:set grafana GF_INSTALL_PLUGINS="grafana-github-datasource"
+dokku config:set grafana GF_FEATURE_TOGGLES_ENABLE="publicDashboards"
+```
+
+## Letsencrypt
+
+```bash
+dokku$ dokku ports:add grafana http:80:3000
+# TODO: block access to port 3000?
+dokku$ dokku letsencrypt:enable grafana
+```
+
+## Create postgresql connection in Grafana
+
+### Configure user on postgresql db cluster
+
+* create `grafanareader` user on cluster in DigitalOcean control panel for primary node
+* by default `grafanareader` cannot connect to `jobserver` database
+* connect to primary node with psql & allow connections:
+
+```sql
+GRANT CONNECT ON database jobserver TO grafanareader;
+```
+
+* `grafanareader` will still fail with e.g. "db query error: pq: permission denied for table applications_application"
+* configure access as required:
+
+```sql
+GRANT SELECT ON applications_application, applications_cmoprioritylistpage, applications_commercialinvolvementpage, applications_datasetspage, applications_legalbasispage, applications_previousehrexperiencepage, applications_recordleveldatapage, applications_referencespage, applications_researcherregistration, applications_sharingcodepage, applications_sponsordetailspage, applications_studydatapage, applications_studyfundingpage, applications_studyinformationpage, applications_studypurposepage, applications_teamdetailspage, applications_typeofstudypage, interactive_analysisrequest, jobserver_backend, jobserver_backendmembership, jobserver_job, jobserver_jobrequest, jobserver_org, jobserver_orgmembership, jobserver_project, jobserver_projectmembership, jobserver_publishrequest, jobserver_release, jobserver_releasefile, jobserver_releasefilereview, jobserver_repo, jobserver_report, jobserver_snapshot, jobserver_snapshot_files, jobserver_stats, jobserver_workspace, redirects_redirect TO grafanareader;
+CREATE VIEW jobserver_user_grafana AS SELECT id,last_login,is_superuser,username,is_staff,is_active,date_joined,fullname,created_by_id,login_token_expires_at,pat_expires_at,roles FROM jobserver_user;
+GRANT SELECT ON jobserver_user_grafana TO grafanareader;
+```
+
+### Connect from Grafana
+
+In the Grafana UI:
+
+* Adminstration
+* Plugins
+* PostgreSQL
+* Add new data source
+* Enter `Host`, `Database`, `User`, `Password` from DigitalOcean db cluster read-only node `Connection Details`
+
+### Missing datasource
+
+If you import a Dashboard from a JSON file, the visualisations may error with "Could not find datasource $UUID". It is sometimes possible to fix this by some combination of hitting "Run query" & "Refresh", otherwise you could recreate the visulation by copy-pasting SQL etc. 

grafana/docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  grafana:
+    # use Dockerfile so that version matches production
+    # uses sqlite for a very simple smoke test 
+    build:
+        dockerfile: Dockerfile
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana:/var/lib/grafana
+
+volumes:
+  grafana:

grafana/justfile

@@ -0,0 +1,21 @@
+
+build:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    test -z "${SKIP_BUILD:-}" || { echo "SKIP_BUILD set"; exit 0; }
+
+    # set build args for prod builds
+    export BUILD_DATE=$(date -u +'%y-%m-%dT%H:%M:%SZ')
+    export GITREF=$(git rev-parse --short HEAD)
+
+    docker compose build grafana
+
+serve:
+    docker compose up grafana -d
+
+# run a basic functional smoke test against a running server
+smoke-test host="http://127.0.0.1:3000/login":
+    #!/bin/bash
+    set -eu
+    curl -I {{ host }} -s --compressed --fail --retry 20 --retry-delay 1 --retry-all-errors