diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5a60f87a..e64fcbb5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,8 @@ updates: directory: "/" schedule: interval: "monthly" + + - package-ecosystem: "docker" + directory: "grafana/" + schedule: + interval: "weekly" diff --git a/.github/workflows/grafana.yml b/.github/workflows/grafana.yml new file mode 100644 index 00000000..d0394868 --- /dev/null +++ b/.github/workflows/grafana.yml @@ -0,0 +1,112 @@ +--- +name: Grafana deployment + +env: + IMAGE_NAME: grafana + PUBLIC_IMAGE_NAME: ghcr.io/ebmdatalab/grafana + REGISTRY: ghcr.io + SSH_AUTH_SOCK: /tmp/agent.sock + +on: + push: + +jobs: + lint-dockerfile: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + with: + dockerfile: grafana/Dockerfile + + docker-test-and-build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: "opensafely-core/setup-action@v1" + with: + install-just: true + + - name: Build docker image + run: | + just grafana/build + + - name: Run smoke test + run: | + just grafana/serve + sleep 5 + just grafana/smoke-test || { docker logs grafana-grafana_1; exit 1; } + + - name: Save docker image + run: | + docker save grafana | gzip > /tmp/grafana.tar.gz + + - name: Upload docker image + uses: actions/upload-artifact@v3 + with: + name: grafana-image + path: /tmp/grafana.tar.gz + + required-checks: + if: always() + + needs: + - docker-test-and-build + - lint-dockerfile + + runs-on: Ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} + + deploy: + needs: [required-checks] + + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + if: github.ref == 'refs/heads/main' + + concurrency: deploy-production + + steps: + - uses: actions/checkout@v4 + - uses: "opensafely-core/setup-action@v1" + with: + install-just: true + + - name: Download docker image + uses: actions/download-artifact@v3 + with: + name: grafana-image + path: /tmp/image + + - name: Import docker image + run: gunzip -c /tmp/image/grafana.tar.gz | docker load + + - name: Test image we imported from previous job works + run: | + SKIP_BUILD=1 just grafana/serve + sleep 5 + just grafana/smoke-test || { docker logs grafana-grafana_1; exit 1; } + + - name: Publish image + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login "$REGISTRY" -u ${{ github.actor }} --password-stdin + docker tag "$IMAGE_NAME" "$PUBLIC_IMAGE_NAME":latest + docker push "$PUBLIC_IMAGE_NAME":latest + + - name: Deploy image + run: | + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "${{ secrets.DOKKU3_DEPLOY_SSH_KEY }}" + SHA=$(docker inspect --format='{{index .RepoDigests 0}}' "$PUBLIC_IMAGE_NAME":latest) + ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" dokku@dokku3.ebmdatalab.net git:from-image grafana "$SHA" diff --git a/docker-compose.yaml b/docker-compose.yaml index 12d97e92..073c831c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,7 +11,9 @@ services: - postgres:/var/lib/postgresql/data/ grafana: - image: "grafana/grafana:latest" + # use Dockerfile so that version matches production + build: + dockerfile: grafana/Dockerfile environment: GF_DATABASE_TYPE: postgres GF_DATABASE_HOST: db:5432 diff --git a/grafana/Dockerfile b/grafana/Dockerfile new file mode 100644 index 00000000..5d59ea36 --- /dev/null +++ b/grafana/Dockerfile @@ -0,0 +1,3 @@ +FROM grafana/grafana-enterprise:latest@sha256:0d33bf1b2c3d508c5345ca652a96676c119420c4474e90299e4703ee834ade70 + +USER grafana diff --git a/grafana/README.md b/grafana/README.md new file mode 100644 index 00000000..47e02c52 --- /dev/null +++ b/grafana/README.md @@ -0,0 +1,84 @@ +# Grafana deployment instructions +## Create app + +```bash +dokku$ dokku apps:create grafana +dokku$ dokku domains:add grafana dashboards.opensafely.org +``` + +## Create postgresql db for grafana + +* on DO db cluster + * postgresql version to match target server - currently 14 on dokku3 + +## create persistent storage + +```bash +# the grafana container runs as uid 472 (grafana) +# the other dokku containers on dokku3 run as uid 1013 (dokku) +# let's tell the container to run as 1013, then we can use the same file permissions +dokku$ dokku docker-options:add grafana deploy "--user 1013" +dokku$ dokku docker-options:add grafana run "--user 1013" +dokku$ dokku storage:ensure-directory grafana + +myuser$ sudo chown -R dokku:dokku /var/lib/dokku/data/storage/grafana + +dokku$ dokku storage:mount grafana /var/lib/dokku/data/storage/grafana:/var/lib/grafana +``` + +## Configure app + +```bash +dokku config:set grafana GF_DATABASE_TYPE="postgres" +dokku config:set grafana GF_DATABASE_HOST="xxx:5432" +dokku config:set grafana GF_DATABASE_NAME="grafana" +dokku config:set grafana GF_DATABASE_USER="grafana" +dokku config:set grafana GF_DATABASE_PASSWORD="xxx" +dokku config:set grafana GF_DATABASE_SSL_MODE="require" +dokku config:set grafana GF_SERVER_ROOT_URL="https://dashboards.opensafely.org/" +dokku config:set grafana GF_INSTALL_PLUGINS="grafana-github-datasource" +dokku config:set grafana GF_FEATURE_TOGGLES_ENABLE="publicDashboards" +``` + +## Letsencrypt + +```bash +dokku$ dokku ports:add grafana http:80:3000 +# TODO: block access to port 3000? +dokku$ dokku letsencrypt:enable grafana +``` + +## Create postgresql connection in Grafana + +### Configure user on postgresql db cluster + +* create `grafanareader` user on cluster in DigitalOcean control panel for primary node +* by default `grafanareader` cannot connect to `jobserver` database +* connect to primary node with psql & allow connections: + +```sql +GRANT CONNECT ON database jobserver TO grafanareader; +``` + +* `grafanareader` will still fail with e.g. "db query error: pq: permission denied for table applications_application" +* configure access as required: + +```sql +GRANT SELECT ON applications_application, applications_cmoprioritylistpage, applications_commercialinvolvementpage, applications_datasetspage, applications_legalbasispage, applications_previousehrexperiencepage, applications_recordleveldatapage, applications_referencespage, applications_researcherregistration, applications_sharingcodepage, applications_sponsordetailspage, applications_studydatapage, applications_studyfundingpage, applications_studyinformationpage, applications_studypurposepage, applications_teamdetailspage, applications_typeofstudypage, interactive_analysisrequest, jobserver_backend, jobserver_backendmembership, jobserver_job, jobserver_jobrequest, jobserver_org, jobserver_orgmembership, jobserver_project, jobserver_projectmembership, jobserver_publishrequest, jobserver_release, jobserver_releasefile, jobserver_releasefilereview, jobserver_repo, jobserver_report, jobserver_snapshot, jobserver_snapshot_files, jobserver_stats, jobserver_workspace, redirects_redirect TO grafanareader; +CREATE VIEW jobserver_user_grafana AS SELECT id,last_login,is_superuser,username,is_staff,is_active,date_joined,fullname,created_by_id,login_token_expires_at,pat_expires_at,roles FROM jobserver_user; +GRANT SELECT ON jobserver_user_grafana TO grafanareader; +``` + +### Connect from Grafana + +In the Grafana UI: + +* Adminstration +* Plugins +* PostgreSQL +* Add new data source +* Enter `Host`, `Database`, `User`, `Password` from DigitalOcean db cluster read-only node `Connection Details` + +### Missing datasource + +If you import a Dashboard from a JSON file, the visualisations may error with "Could not find datasource $UUID". It is sometimes possible to fix this by some combination of hitting "Run query" & "Refresh", otherwise you could recreate the visulation by copy-pasting SQL etc. diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml new file mode 100644 index 00000000..12aca85f --- /dev/null +++ b/grafana/docker-compose.yml @@ -0,0 +1,13 @@ +services: + grafana: + # use Dockerfile so that version matches production + # uses sqlite for a very simple smoke test + build: + dockerfile: Dockerfile + ports: + - 3000:3000 + volumes: + - grafana:/var/lib/grafana + +volumes: + grafana: diff --git a/grafana/justfile b/grafana/justfile new file mode 100644 index 00000000..e8292f4b --- /dev/null +++ b/grafana/justfile @@ -0,0 +1,21 @@ + +build: + #!/usr/bin/env bash + set -euo pipefail + + test -z "${SKIP_BUILD:-}" || { echo "SKIP_BUILD set"; exit 0; } + + # set build args for prod builds + export BUILD_DATE=$(date -u +'%y-%m-%dT%H:%M:%SZ') + export GITREF=$(git rev-parse --short HEAD) + + docker compose build grafana + +serve: + docker compose up grafana -d + +# run a basic functional smoke test against a running server +smoke-test host="http://127.0.0.1:3000/login": + #!/bin/bash + set -eu + curl -I {{ host }} -s --compressed --fail --retry 20 --retry-delay 1 --retry-all-errors