Merge pull request #3 from dbca-wa/develop

[6346] Create and configure CKAN 2.10 project

.ahoy.yml

@@ -0,0 +1,152 @@
+---
+ahoyapi: v2
+
+entrypoint:
+  - bash
+  - "-c"
+  - "-e"
+  - |
+    [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+    bash -e -c "$0" "$@"
+  - "{{cmd}}"
+  - "{{name}}"
+
+commands:
+  init:
+    usage: Initialise the codebase on first-time setup (ahoy init)
+    cmd : |
+      echo "Creating project variables."
+      cp .env.dbca .env
+
+      echo "Install local development extensions."
+      sh src/dbca_install_extensions.sh
+  up:
+    usage: Build project.
+    cmd: |
+      docker compose -f $DOCKER_COMPOSE up -d "$@"
+      ahoy info;
+
+  down:
+    usage: Delete project (CAUTION).
+    cmd: |
+      if [ "$1" == "y" ]; then
+        docker compose -f $DOCKER_COMPOSE down --volumes
+      else
+        ahoy confirm "Running this command will destroy your current site, database and build? Are you sure you didn't mean ahoy stop?" &&
+        # Run this if confirm returns true
+        docker compose -f $DOCKER_COMPOSE down --volumes ||
+        # Run this if confirm returns false
+        echo "OK, probably a wise choice..."
+      fi
+
+  build:
+    usage: Build project.
+    cmd: |
+      if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+        CKAN_CONTAINER_NAME=ckan-dev
+        WORKER_CONTAINER_NAME=ckan-dev-worker
+      fi
+      docker compose -f $DOCKER_COMPOSE build "$@"
+
+  cli:
+    usage: Start a shell inside container.
+    cmd: |
+      if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+        CKAN_CONTAINER_NAME=ckan-dev
+      fi
+      docker compose -f $DOCKER_COMPOSE exec ${1:-$CKAN_CONTAINER_NAME} sh
+
+  run:
+    usage: Run command inside container.
+    cmd: |
+      if [[ $# -eq 2 ]]; then
+        SERVICE=$1
+        COMMAND=$2
+      else
+        if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+          CKAN_CONTAINER_NAME=ckan-dev
+        fi
+        SERVICE=$CKAN_CONTAINER_NAME
+        COMMAND=$1
+      fi;
+      docker compose -f $DOCKER_COMPOSE exec -T $SERVICE sh -c "$COMMAND"
+
+  logs:
+    usage: Show Docker logs.
+    cmd: |
+      if [[ $# -eq 2 ]]; then
+        SERVICE=$1
+        TAIL=$2
+      else
+        SERVICE=$1
+        TAIL=100
+      fi;
+      docker compose -f $DOCKER_COMPOSE logs -f --tail=$TAIL $SERVICE
+
+  ps:
+    usage: List running Docker containers.
+    cmd: docker compose -f $DOCKER_COMPOSE ps
+
+  restart:
+    usage: Restart Docker containers.
+    cmd: |
+      docker compose -f $DOCKER_COMPOSE restart "$@"
+      ahoy info;
+
+  stop:
+    usage: Stop Docker containers.
+    cmd: docker compose -f $DOCKER_COMPOSE stop "$@"
+
+  attach:
+    usage: Attach to a running container
+    cmd: docker attach $(docker compose -f $DOCKER_COMPOSE ps -q "$@")
+
+  recreate:
+    usage: Recreate a local container | ahoy recreate ckan
+    cmd: |
+      docker compose -f $DOCKER_COMPOSE up -d --force-recreate --no-deps --build "${1}"
+
+  info:
+    usage: Print information about this project.
+    cmd: |
+      if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+        SITE_URL="http://localhost:${CKAN_PORT_HOST}"
+      else
+        SITE_URL="https://localhost:${NGINX_SSLPORT_HOST}"
+      fi
+      echo "Site local URL        : ${SITE_URL}"
+      if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+        echo "CKAN DB port on host  : $(docker port $(docker compose -f $DOCKER_COMPOSE ps -q $POSTGRESQL_CONTAINER_NAME) 5432 | cut -d : -f 2)"
+        echo "SOLR port on host     : $(docker port $(docker compose -f $DOCKER_COMPOSE ps -q $SOLR_CONTAINER_NAME) 8983 | cut -d : -f 2)"
+      fi
+
+  db-import:
+    usage: Pipe in a postgres dump file.  `ahoy db-import local.dump`
+    cmd: |
+      if [ -e "$@" ] ; then
+        docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v' < "$@"
+      else echo "Provided sql file" "$@" "does not exist"
+      fi
+
+  db-dump:
+    usage: Dump data out into a file. `ahoy db-dump local.dump`
+    cmd: docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_dump -U $CKAN_DB_USER -d $CKAN_DB --format=custom -v' > "$@"
+
+  confirm:
+    cmd: read -r -p "${@} [y/N] " response; [ ${response} = "y" ]
+    hide: true
+
+  generate-extension:
+    usage: Generates a new CKAN extension into the src directory
+    cmd: ahoy run cli 'ckan generate extension -o ${SRC_DIR}'
+
+  open:
+    usage: Open the site in your default browser
+    cmd: |
+      if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+        SITE_URL="http://localhost:${CKAN_PORT_HOST}"
+      else
+        SITE_URL="http://localhost:${NGINX_PORT_HOST}"
+      fi
+      open $SITE_URL
+

.devcontainer/devcontainer.json

@@ -0,0 +1,82 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+	"name": "CKAN DBCA 2.10",
+
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../docker-compose.dev.yml",
+		"docker-compose.yml"
+	],
+
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "ckan-dev",
+
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/srv",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"ms-python.autopep8",
+				"ms-python.flake8",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"GitHub.vscode-pull-request-github",
+				"eamodio.gitlens",
+				"streetsidesoftware.code-spell-checker",
+				"wholroyd.jinja",
+				"GitHub.copilot",
+				"GitHub.copilot-chat"
+			],
+			// Set *default* container specific settings.json values on container create.
+			"settings": {
+				"terminal.integrated.profiles.linux": {
+					"bash": {
+						"path": "/bin/bash"
+					}
+				},
+				"python.languageServer": "Pylance",
+				"python.linting.enabled": true,
+				"python.linting.flake8Enabled": true,
+				"flake8.args": [
+					"--max-line-length",
+					"120"
+				],
+				"python.formatting.provider": "autopep8",
+				"autopep8.args": [
+					"--max-line-length",
+					"120"
+				],
+				"cSpell.language": "en-GB",
+				"python.analysis.extraPaths": [
+					"./app/src"
+				]
+			}
+		}
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	// "shutdownAction": "none",
+
+	// Uncomment the next line to run commands after the container is created.
+	"postStartCommand": "${APP_DIR}/start_ckan_development.sh",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  ckan-dev:
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer 
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary* 
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    #
+    # build:
+    #   context: .
+    #   dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - .:/srv/workspace:cached
+      - .vscode:/srv/.vscode:cached
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: /bin/sh -c "while sleep 1000; do :; done"

.env.dbca

@@ -0,0 +1,87 @@
+# Container names
+NGINX_CONTAINER_NAME=nginx
+REDIS_CONTAINER_NAME=redis
+POSTGRESQL_CONTAINER_NAME=db
+SOLR_CONTAINER_NAME=solr
+DATAPUSHER_CONTAINER_NAME=datapusher
+CKAN_CONTAINER_NAME=ckan
+WORKER_CONTAINER_NAME=ckan-worker
+
+# Host Ports
+CKAN_PORT_HOST=5000
+NGINX_PORT_HOST=81
+NGINX_SSLPORT_HOST=8443
+
+# CKAN databases
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+POSTGRES_HOST=db
+CKAN_DB_USER=ckandbuser
+CKAN_DB_PASSWORD=ckandbpassword
+CKAN_DB=ckandb
+DATASTORE_READONLY_USER=datastore_ro
+DATASTORE_READONLY_PASSWORD=datastore
+DATASTORE_DB=datastore
+CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
+CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+
+# Dev settings
+USE_HTTPS_FOR_DEV=false
+
+# CKAN core
+CKAN_VERSION=2.10.0
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+
+CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
+# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
+CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
+CKAN_SYSADMIN_NAME=ckan_admin
+CKAN_SYSADMIN_PASSWORD=test1234
+[email protected]
+CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+TZ=UTC
+
+# Solr
+SOLR_IMAGE_VERSION=2.10-solr9
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+
+# Redis
+REDIS_VERSION=7
+CKAN_REDIS_URL=redis://redis:6379/1
+TEST_CKAN_REDIS_URL=redis://redis:6379/1
+
+# Xloader
+CKANEXT__XLOADER__JOBS_DB__URI=$CKAN_SQLALCHEMY_URL
+
+# NGINX
+NGINX_PORT=80
+NGINX_SSLPORT=443
+
+# Extensions
+CKAN__PLUGINS="envvars image_view text_view datatables_view datastore xloader"
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1
+
+## WA DBCA Config ##
+# Docker compose project name
+COMPOSE_PROJECT_NAME=dbca
+# The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds)
+DOCKER_COMPOSE=docker-compose.dev.yml