Merge pull request #5 from dbca-wa/develop

Updated project to use ckan config file

.ahoy.yml

@@ -74,12 +74,12 @@ commands:
   logs:
     usage: Show Docker logs.
     cmd: |
+      TAIL=100
       if [[ $# -eq 2 ]]; then
         SERVICE=$1
         TAIL=$2
       else
         SERVICE=$1
-        TAIL=100
       fi;
       docker compose -f $DOCKER_COMPOSE logs -f --tail=$TAIL $SERVICE
 
@@ -124,7 +124,7 @@ commands:
     usage: Pipe in a postgres dump file.  `ahoy db-import local.dump`
     cmd: |
       if [ -e "$@" ] ; then
-        docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v' < "$@"
+        docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@"
       else echo "Provided sql file" "$@" "does not exist"
       fi
 

.env.dbca

@@ -3,7 +3,6 @@ NGINX_CONTAINER_NAME=nginx
 REDIS_CONTAINER_NAME=redis
 POSTGRESQL_CONTAINER_NAME=db
 SOLR_CONTAINER_NAME=solr
-DATAPUSHER_CONTAINER_NAME=datapusher
 CKAN_CONTAINER_NAME=ckan
 WORKER_CONTAINER_NAME=ckan-worker
 
@@ -74,11 +73,7 @@ NGINX_PORT=80
 NGINX_SSLPORT=443
 
 # Extensions
-CKAN__PLUGINS="envvars image_view text_view datatables_view datastore xloader"
-CKAN__HARVEST__MQ__TYPE=redis
-CKAN__HARVEST__MQ__HOSTNAME=redis
-CKAN__HARVEST__MQ__PORT=6379
-CKAN__HARVEST__MQ__REDIS_DB=1
+
 
 ## WA DBCA Config ##
 # Docker compose project name

.vscode/launch.json

@@ -13,7 +13,7 @@
                 "-c continue",
                 "/usr/bin/ckan",
                 "--config",
-                "/srv/app/ckan.ini",
+                "/srv/app/config/dbca.ini",
                 "run",
                 "--host",
                 "0.0.0.0",
@@ -30,9 +30,24 @@
                 "-c continue",
                 "/usr/bin/ckan",
                 "--config",
-                "/srv/app/ckan.ini",
+                "/srv/app/config/dbca.ini",
                 "jobs",
                 "worker",
+                "priority"
+            ],
+            "justMyCode": true
+        },
+        {
+            "name": "Python: CKAN CLI Command Run & Debug",
+            "type": "python",
+            "request": "launch",
+            "module": "pdb",
+            "args": [
+                "-c continue",
+                "/srv/app/config/dbca.ini",
+                "--config",
+                "/srv/app/dbca.ini",
+                "<CLI_COMMAND>"
             ],
             "justMyCode": true
         }

ckan/Dockerfile

@@ -21,3 +21,15 @@ RUN for d in $APP_DIR/patches/*; do \
             done ; \
         fi ; \
     done
+
+## DBCA specific configuration ##
+
+# Override the default start_ckan.sh script
+COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh
+RUN chmod +x ${APP_DIR}/start_ckan.sh
+
+# Copy config files
+COPY config/*.ini ${APP_DIR}/config/
+
+# Override default CKAN config file to use dbca.ini
+ENV CKAN_INI=${APP_DIR}/config/dbca.ini

ckan/Dockerfile.dev

@@ -38,9 +38,6 @@ RUN pip3 install pip --upgrade && \
     chmod +x ${APP_DIR}/dbca_requirements.sh && \
     ${APP_DIR}/dbca_requirements.sh
 
-COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh
-RUN  chmod +x ${APP_DIR}/start_ckan_development.sh
-
 # Clone the extension(s) your are writing for your own project in the `src` folder
 # to get them mounted in this image at runtime
 
@@ -58,3 +55,15 @@ RUN for d in $APP_DIR/patches/*; do \
             done ; \
         fi ; \
     done
+
+## DBCA specific configuration ##
+
+# Override the default dbca_start_ckan_development.sh script
+COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh
+RUN chmod +x ${APP_DIR}/start_ckan_development.sh
+
+# Copy config files
+COPY config/*.ini ${APP_DIR}/config/
+
+# Override default CKAN config file to use dbca.ini
+ENV CKAN_INI=${APP_DIR}/config/dbca.ini

ckan/config/dbca.ini

@@ -0,0 +1,64 @@
+#
+# DBCA configuration
+#
+
+[app:main]
+
+## General settings ############################################################
+# This will include the config file settings from the default ckan config file
+use = config:/srv/app/ckan.ini
+ckan.devserver.watch_patterns = /srv/app/ckan.ini
+
+## Plugins Settings ############################################################
+ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat envvars
+
+## Resource Views Settings #####################################################
+ckan.views.default_views = text_view datatables_view pdf_view
+
+## Internationalisation Settings ###############################################
+ckan.locale_default = en_AU
+ckan.display_timezone = Australia/West
+
+## CKAN Extensions configuration ###############################################
+
+
+## Logging configuration
+[loggers]
+keys = root, ckan, ckanext, werkzeug
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_werkzeug]
+level = WARNING
+handlers = console
+qualname = werkzeug
+propagate = 0
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

ckan/config/uwsgi.ini

@@ -0,0 +1,16 @@
+[uwsgi]
+
+plugins= http,python
+socket=/tmp/uwsgi.sock
+wsgi-file=/srv/app/wsgi.py
+module=wsgi:application
+uid=92
+gid=92
+http=0.0.0.0:5000
+master=enable-threads
+lazy-apps=true
+processes=5
+disable-logging=true
+buffer-size=32768
+vacuum=true
+harakiri=50

ckan/docker-entrypoint.d/02_setup_dbca.sh

@@ -19,6 +19,6 @@ fi
 #     ckan -c $CKAN_INI db upgrade -p harvest
 # fi
 
-# if [[ $CKAN__PLUGINS == *"pages"* ]]; then
-#     ckan -c $CKAN_INI pages initdb
-# fi
+if [[ $CKAN__PLUGINS == *"pages"* ]]; then
+    ckan -c $CKAN_INI pages initdb
+fi

ckan/setup/dbca_requirements.sh

@@ -20,6 +20,9 @@ pip3 install -r ${SRC_DIR}/ckanext-hierarchy/requirements.txt
 # Pages
 pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-pages
 
+# PDF View
+pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-pdfview
+
 # Report
 pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext-report
 pip3 install -r ${SRC_DIR}/ckanext-report/requirements.txt

ckan/setup/dbca_start_ckan.sh.override

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Update the default ckan config file
+export CKAN_INI=$APP_DIR/ckan.ini
+
+# Set up the Secret key used by Beaker and Flask
+# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
+if grep -qE "beaker.session.secret ?= ?$" ckan.ini
+then
+    echo "Setting beaker.session.secret in ini file"
+    ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
+    JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
+    ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
+    ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
+fi
+
+# Run the prerun script to init CKAN and create the default admin user
+python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; bash "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python3 "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+    done
+fi
+
+# Revert back touse DBCA config file
+export CKAN_INI=$APP_DIR/config/dbca.ini
+# Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file
+unset CKAN__PLUGINS
+
+if [ $? -eq 0 ]
+then
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    uwsgi -i $APP_DIR/config/uwsgi.ini
+else
+  echo "[prerun] failed...not starting CKAN."
+fi

ckan/setup/dbca_start_ckan_development.sh.override

@@ -42,7 +42,8 @@ if [ ! -f /tmp/container_ready ]; then
             fi
         fi
     done
-
+    # Update the default ckan config file
+    export CKAN_INI=$APP_DIR/ckan.ini
     # Set debug to true
     echo "Enabling debug mode"
     ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
@@ -88,6 +89,10 @@ if [ ! -f /tmp/container_ready ]; then
         done
     fi
 
+    # Revert back touse DBCA config file
+    export CKAN_INI=$APP_DIR/config/dbca.ini
+    # Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file
+    unset CKAN__PLUGINS
     # Set the container as ready so the startup scripts are not run again
     touch /tmp/container_ready
 fi

docker-compose.dev.yml

@@ -31,6 +31,7 @@ services:
     volumes:
       - ckan_storage:/var/lib/ckan
       - ./src:/srv/app/src_extensions
+      - ./ckan/config:/srv/app/config
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]

docker-compose.yml

@@ -16,6 +16,8 @@ services:
     networks:
       - webnet
       - ckannet
+    env_file:
+      - .env
     depends_on:
       ckan:
         condition: service_healthy

nginx/Dockerfile

@@ -9,6 +9,7 @@ RUN apk update --no-cache && \
 COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf
 COPY setup/index.html /usr/share/nginx/html/index.html
 COPY setup/default.conf ${NGINX_DIR}/conf.d/
+COPY setup/default.conf.template ${NGINX_DIR}/templates/
 
 RUN mkdir -p ${NGINX_DIR}/certs
 

nginx/setup/default.conf.template

@@ -0,0 +1,46 @@
+server {
+    #listen       80;
+    #listen  [::]:80;
+    listen       443 ssl;
+    listen  [::]:443 ssl;
+    server_name  localhost;
+    ssl_certificate /etc/nginx/certs/ckan-local.crt;
+    ssl_certificate_key /etc/nginx/certs/ckan-local.key;
+
+    # TLS 1.2 & 1.3 only
+    ssl_protocols TLSv1.2 TLSv1.3;
+
+    # Disable weak ciphers
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+
+    # SSL sessions
+    ssl_session_timeout 1d;
+    # ssl_session_cache dfine in stream and http
+    ssl_session_tickets off;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        proxy_pass http://${CKAN_CONTAINER_NAME:-ckan}:${CKAN_PORT:-5000}/;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header Host $host;
+        #proxy_cache cache;
+        proxy_cache_bypass $cookie_auth_tkt;
+        proxy_no_cache $cookie_auth_tkt;
+        proxy_cache_valid 30m;
+        proxy_cache_key $host$scheme$proxy_host$request_uri;
+    }
+
+    error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html;
+
+    # redirect server error pages to the static page /error.html
+    #
+    location = /error.html {
+      ssi on;
+      internal;
+      auth_basic off;
+      root /usr/share/nginx/html;
+    }
+
+}

src/dbca_install_extensions.sh

@@ -24,4 +24,4 @@ cd src/
 # Featured Views
 # git clone https://github.com/dbca-wa/ckanext-featuredviews
 
-echo "Ready to build project: ahoy build"
+echo "Ready to build project: ahoy build"