Feature/6374 qa extension upgrade (#2)

* Installed QA extension Enabled plugins archiver & report Added archiver config values Initialise db for archiver and report Added bulk & priority background jobs for archiver Added cron job for report generation Added nginx config to serve archiver cached resources Added ckan_storage volume to nginx to access cached resources * Install qsv dependency for extension ckanext-qa * Simplified installation of qsv * Fixed issues with env variable CKAN__PLUGINS using the value from base image * Added qa dependency file package Updated qa extension to develop branch
dbca-wa · Jan 31, 2024 · b03f0d5 · b03f0d5
1 parent ca5a331
commit b03f0d5
Show file tree

Hide file tree

Showing 17 changed files with 157 additions and 49 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ _solr/schema.xml
 _src/*
 local/*
 .env
+dbca/
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -32,8 +32,7 @@
                 "--config",
                 "/srv/app/config/dbca.ini",
                 "jobs",
-                "worker",
-                "priority"
+                "worker"
             ],
             "justMyCode": true
         },
@@ -44,9 +43,9 @@
             "module": "pdb",
             "args": [
                 "-c continue",
-                "/srv/app/config/dbca.ini",
+                "/usr/bin/ckan",
                 "--config",
-                "/srv/app/dbca.ini",
+                "/srv/app/config/dbca.ini",
                 "<CLI_COMMAND>"
             ],
             "justMyCode": true

diff --git a/ckan/Dockerfile b/ckan/Dockerfile
@@ -32,4 +32,4 @@ RUN chmod +x ${APP_DIR}/start_ckan.sh
 COPY config/*.ini ${APP_DIR}/config/
 
 # Override default CKAN config file to use dbca.ini
-ENV CKAN_INI=${APP_DIR}/config/dbca.ini
+ENV CKAN_INI=${APP_DIR}/config/dbca.ini
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
@@ -31,7 +31,6 @@ FROM ckan/ckan-dev:2.10.3
 #RUN  pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat && \
 #     pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
 
-
 # Install any extensions needed by your CKAN instance
 COPY setup/dbca_requirements.sh ${APP_DIR}
 RUN pip3 install pip --upgrade && \
@@ -66,4 +65,4 @@ RUN chmod +x ${APP_DIR}/start_ckan_development.sh
 COPY config/*.ini ${APP_DIR}/config/
 
 # Override default CKAN config file to use dbca.ini
-ENV CKAN_INI=${APP_DIR}/config/dbca.ini
+ENV CKAN_INI=${APP_DIR}/config/dbca.ini
diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini
@@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini
 ckan.devserver.watch_patterns = /srv/app/ckan.ini
 
 ## Plugins Settings ############################################################
-ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi resource_proxy geo_view shp_view envvars
+ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view envvars
 
 ## Resource Views Settings #####################################################
 ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view
@@ -44,6 +44,14 @@ ckanext.showcase.editor = ckeditor
 ckanext.pages.allow_html = True
 ckanext.pages.editor = ckeditor
 
+# ckanext-archiver
+ckanext-archiver.archive_dir=/var/lib/ckan/archiver
+ckanext-archiver.cache_url_root=/resource_cache/
+
+# ckanext-qa
+ckanext.qa.qsv_bin=/usr/local/bin/qsv
+
+
 ## Logging configuration
 [loggers]
 keys = root, ckan, ckanext, werkzeug

diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-# Get the ckan plugins values from the DBCA CKAN config file
-CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2)
 echo "CKAN__PLUGINS: $CKAN__PLUGINS"
 
 if [[ $CKAN__PLUGINS == *"xloader"* ]]; then
@@ -10,22 +8,19 @@ if [[ $CKAN__PLUGINS == *"xloader"* ]]; then
     echo "Setting a temporary value for ckanext.xloader.api_token"
     ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')"
 fi
+CKAN_INI=$APP_DIR/config/dbca.ini 
 
-# Use the DBCA CKAN config file for the CKAN config file
-CKAN_INI=$APP_DIR/config/dbca.ini
-
-## Examples of how to initialise DB for the extensions
-# if [[ $CKAN__PLUGINS == *"archiver"* ]]; then
-#     ckan -c $CKAN_INI archiver init
-# fi
+if [[ $CKAN__PLUGINS == *"archiver"* ]]; then
+    ckan -c $CKAN_INI archiver init
+fi
 
-# if [[ $CKAN__PLUGINS == *"report"* ]]; then
-#     ckan -c $CKAN_INI report initdb
-# fi
+if [[ $CKAN__PLUGINS == *"report"* ]]; then
+    ckan -c $CKAN_INI report initdb
+fi
 
-# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then
-#     ckan -c $CKAN_INI db upgrade -p harvest
-# fi
+if [[ $CKAN__PLUGINS == *"qa"* ]]; then
+    ckan -c $CKAN_INI qa init
+fi
 
 if [[ $CKAN__PLUGINS == *"pages"* ]]; then
     ckan -c $CKAN_INI pages initdb
@@ -34,3 +29,7 @@ fi
 if [[ $CKAN__PLUGINS == *"doi"* ]]; then
     ckan -c $CKAN_INI doi initdb
 fi
+
+# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then
+#     ckan -c $CKAN_INI db upgrade -p harvest
+# fi
diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs
@@ -1,4 +1,7 @@
 # Crontab for CKAN cron jobs
 # Example cron job runs the harvester run command every 15 mins
-#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+# Midnight task to schedule embargo datasets to public visibility
 0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+# Midnight report generation for archiver broken links
+0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini report generate >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh
@@ -1,16 +1,17 @@
 #!/bin/sh
 
-## Must Have ##
+## CKAN Core extensions ##
+
 # Archiver
-pip3 install -e 'git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver'
+pip3 install -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver
 pip3 install -r ${SRC_DIR}/ckanext-archiver/requirements.txt
 
 # DCAT
 pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat
 pip3 install -r ${SRC_DIR}/ckanext-dcat/requirements.txt
 
 # Harvester
-pip3 install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-harvest'
+pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-harvest
 pip3 install -r ${SRC_DIR}/ckanext-harvest/requirements.txt
 
 # Hierarchy
@@ -32,7 +33,7 @@ pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckan
 pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt
 
 # Scheming
-pip3 install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-scheming'
+pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-scheming
 
 # Spatial
 # dependencies
@@ -46,15 +47,28 @@ pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckane
 pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt
 
 # XLoader
-pip3 install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-xloader'
+pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-xloader
 pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt
 
 # Geoview
-pip3 install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-geoview'
+pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-geoview
+
 
-# 3rd Party #
+## 3rd Party ##
 # DOI
 pip3 install -e git+https://github.com/NaturalHistoryMuseum/[email protected]#egg=ckanext-doi
 
-# DBCA Project
+
+## DBCA Project ##
+
+# DBCA
 pip3 install -e git+https://github.com/dbca-wa/ckanext-dbca.git@develop#egg=ckanext-dbca
+
+# QA
+# Install qsv dependency for extension ckanext-qa
+wget -O /tmp/qsv.zip https://github.com/jqnatividad/qsv/releases/download/0.110.0/qsv-0.110.0-x86_64-unknown-linux-musl.zip
+unzip /tmp/qsv.zip -d /usr/local/bin
+rm /tmp/qsv.zip
+apk add file
+pip3 install -e git+https://github.com/dbca-wa/ckanext-qa.git@develop#egg=ckanext-qa
+pip3 install -r ${SRC_DIR}/ckanext-qa/requirements.txt
diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# Set the CKAN plugins variable to the plugins value from the dbca.ini config file
+export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2)
 # Update the default ckan config file
 export CKAN_INI=$APP_DIR/ckan.ini
 
@@ -32,8 +34,6 @@ fi
 
 # Revert back touse DBCA config file
 export CKAN_INI=$APP_DIR/config/dbca.ini
-# Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file
-unset CKAN__PLUGINS
 
 if [ $? -eq 0 ]
 then

diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override
@@ -1,5 +1,8 @@
 #!/bin/sh
 
+# Set the CKAN plugins variable to the plugins value from the dbca.ini config file
+export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2)
+
 # Only run these start up scripts the first time the container is created
 if [ ! -f /tmp/container_ready ]; then
     # Install any local extensions in the src_extensions volume
@@ -60,10 +63,6 @@ if [ ! -f /tmp/container_ready ]; then
         ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
     fi
 
-    # Update the plugins setting in the ini file with the values defined in the env var
-    echo "Loading the following plugins: $CKAN__PLUGINS"
-    ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
-
     # Update test-core.ini DB, SOLR & Redis settings
     echo "Loading test settings into test-core.ini"
     ckan config-tool $SRC_DIR/ckan/test-core.ini \
@@ -91,8 +90,7 @@ if [ ! -f /tmp/container_ready ]; then
 
     # Revert back touse DBCA config file
     export CKAN_INI=$APP_DIR/config/dbca.ini
-    # Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file
-    unset CKAN__PLUGINS
+
     # Set the container as ready so the startup scripts are not run again
     touch /tmp/container_ready
 fi

diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf
@@ -0,0 +1,42 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+
+; 1. Copy this file to /etc/supervisor/conf.d
+; 2. Make sure the paths below match your setup
+
+
+[program:ckan-worker-bulk]
+
+; Use the full paths to the virtualenv and your configuration file here.
+command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk"
+
+
+; User the worker runs as.
+user=ckan
+
+
+; Start just a single worker. Increase this number if you have many or
+; particularly long running background jobs.
+numprocs=1
+process_name=%(program_name)s-%(process_num)02d
+
+
+; Log files.
+stdout_logfile=/srv/app/logs/ckan-worker.stdout.log
+stderr_logfile=/srv/app/logs/ckan-worker.stderr.log
+
+
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+
+
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf
@@ -9,7 +9,7 @@
 [program:ckan-worker-default]
 
 ; Use the full paths to the virtualenv and your configuration file here.
-command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker
+command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker"
 
 
 ; User the worker runs as.

diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf
@@ -0,0 +1,42 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+
+; 1. Copy this file to /etc/supervisor/conf.d
+; 2. Make sure the paths below match your setup
+
+
+[program:ckan-worker-priority]
+
+; Use the full paths to the virtualenv and your configuration file here.
+command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority"
+
+
+; User the worker runs as.
+user=ckan
+
+
+; Start just a single worker. Increase this number if you have many or
+; particularly long running background jobs.
+numprocs=1
+process_name=%(program_name)s-%(process_num)02d
+
+
+; Log files.
+stdout_logfile=/srv/app/logs/ckan-worker.stdout.log
+stderr_logfile=/srv/app/logs/ckan-worker.stderr.log
+
+
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+
+
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -23,6 +23,8 @@ services:
         condition: service_healthy
     ports:
       - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}"
+    volumes:
+      - ckan_storage:/var/lib/ckan
 
   ckan:
     container_name: ${CKAN_CONTAINER_NAME}

diff --git a/nginx/Dockerfile b/nginx/Dockerfile
@@ -21,4 +21,4 @@ ENTRYPOINT \
     -keyout ${NGINX_DIR}/certs/ckan-local.key \
     -out ${NGINX_DIR}/certs/ckan-local.crt \
     -days 365 && \
-  /docker-entrypoint.sh nginx -g 'daemon off;'
+  /docker-entrypoint.sh nginx -g 'daemon off;'
diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf
@@ -32,6 +32,10 @@ server {
         proxy_cache_key $host$scheme$proxy_host$request_uri;
     }
 
+    # archived files from ckanext-archiver
+    location /resource_cache/ {
+      alias /var/lib/ckan/archiver/;
+    }
     error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html;
 
     # redirect server error pages to the static page /error.html

diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh
@@ -5,16 +5,13 @@
 # Uncomment the following lines to install these extension you are working on to upgrade to CKAN 2.10
 
 cd src/
-
-## Must Have ##
+# DBCA
 git clone https://github.com/dbca-wa/ckanext-dbca.git
-
 # QA
-# git clone https://github.com/dbca-wa/ckanext-qa.git
-# These extensions will be installed by default, but we don't want them
-# sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt
+git clone https://github.com/dbca-wa/ckanext-qa.git
+#These extensions will be installed by default, but we don't want them
+sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt
 # Office Docs
 # git clone https://github.com/dbca-wa/ckanext-officedocs
 
-
 echo "Ready to build project: ahoy build"
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,3 +8,4 @@ _solr/schema.xml @@
     _src/*
     local/*
     .env
+    dbca/