From 0a15bea8f84a1989e73ebd2dcd5b1b8099267dfc Mon Sep 17 00:00:00 2001 From: awang Date: Fri, 8 Dec 2023 16:17:28 +0700 Subject: [PATCH 001/123] [BIO-6387] enabled dbca extension and add custom resource formats --- .env.dbca | 5 ++++- src/dbca_install_extensions.sh | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.env.dbca b/.env.dbca index 10d77393..915b3c82 100644 --- a/.env.dbca +++ b/.env.dbca @@ -74,7 +74,7 @@ NGINX_PORT=80 NGINX_SSLPORT=443 # Extensions -CKAN__PLUGINS="image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat envvars" +CKAN__PLUGINS="image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca envvars" CKAN__VIEWS__DEFAULT_VIEWS="text_view datatables_view pdf_view" CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis @@ -86,3 +86,6 @@ CKAN__HARVEST__MQ__REDIS_DB=1 COMPOSE_PROJECT_NAME=dbca # The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds) DOCKER_COMPOSE=docker-compose.dev.yml + +## Internationalisation Settings +CKAN__RESOURCE_FORMATS=/srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config/resource_formats.json diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index 3538b477..6bb6711e 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -7,6 +7,8 @@ cd src/ ## Must Have ## +git clone https://github.com/salsadigitalauorg/ckanext-dbca.git + # QA # git clone https://github.com/dbca-wa/ckanext-qa.git # These extensions will be installed by default, but we don't want them From 958a0ca50381aa6ca657e8f204c154c9ad73d4b8 Mon Sep 17 00:00:00 2001 From: awang Date: Sat, 9 Dec 2023 17:14:28 +0700 Subject: [PATCH 002/123] [BIO-6392] enabled scheming dataset and set timezone to Perth --- .env.dbca | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.env.dbca b/.env.dbca index 915b3c82..0abb4f92 100644 --- a/.env.dbca +++ b/.env.dbca @@ -54,7 +54,7 @@ CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost -TZ=UTC +TZ=Australia/Perth # Solr SOLR_IMAGE_VERSION=2.10-solr9 @@ -74,7 +74,7 @@ NGINX_PORT=80 NGINX_SSLPORT=443 # Extensions -CKAN__PLUGINS="image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca envvars" +CKAN__PLUGINS="image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets envvars" CKAN__VIEWS__DEFAULT_VIEWS="text_view datatables_view pdf_view" CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis @@ -87,5 +87,10 @@ COMPOSE_PROJECT_NAME=dbca # The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds) DOCKER_COMPOSE=docker-compose.dev.yml -## Internationalisation Settings +# Internationalisation Settings +CKAN__DISPLAY_TIMEZONE=Australia/Perth CKAN__RESOURCE_FORMATS=/srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config/resource_formats.json + +# Scheming +CKAN___SCHEMING__DATASET_SCHEMAS=ckanext.dbca:dbca_dataset.yaml +CKAN___SCHEMING__PRESETS="ckanext.scheming:presets.json ckanext.dbca:dbca_presets.json" From 2fdefc06d0b9685795eb340e0c498935cf90ef59 Mon Sep 17 00:00:00 2001 From: awang Date: Wed, 13 Dec 2023 18:37:20 +0700 Subject: [PATCH 003/123] [BIO-6392] added cronjob for auto publish and email notification --- ckan/setup/dbca_ckan_cron_jobs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index a4bec897..3a6bd0e0 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,3 +1,4 @@ # Crontab for CKAN cron jobs # Example cron job runs the harvester run command every 15 mins -#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file +#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 From bd9c45556e59fdcae54484d4a9c03b5f6fa5290e Mon Sep 17 00:00:00 2001 From: awang Date: Thu, 14 Dec 2023 09:24:53 +0700 Subject: [PATCH 004/123] [BIO-6392] updated repo --- src/dbca_install_extensions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index 6bb6711e..bc6a718e 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -7,7 +7,7 @@ cd src/ ## Must Have ## -git clone https://github.com/salsadigitalauorg/ckanext-dbca.git +git clone https://github.com/dbca-wa/ckanext-dbca.git # QA # git clone https://github.com/dbca-wa/ckanext-qa.git From 90c070a3472a33b6064a1c1b28feb40f59b69df3 Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:59:43 +1300 Subject: [PATCH 005/123] Merge branch 'develop' into feature/BIO-6387-6392 --- .ahoy.yml | 4 +- .env.dbca | 1 - .vscode/launch.json | 19 +++++- ckan/Dockerfile | 14 +++- ckan/Dockerfile.dev | 17 +++-- ckan/config/dbca.ini | 64 +++++++++++++++++++ ckan/config/uwsgi.ini | 16 +++++ ckan/setup/dbca_start_ckan.sh.override | 46 +++++++++++++ .../dbca_start_ckan_development.sh.override | 7 +- docker-compose.dev.yml | 1 + docker-compose.yml | 2 + nginx/Dockerfile | 1 + nginx/setup/default.conf.template | 46 +++++++++++++ 13 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 ckan/config/dbca.ini create mode 100644 ckan/config/uwsgi.ini create mode 100755 ckan/setup/dbca_start_ckan.sh.override create mode 100644 nginx/setup/default.conf.template diff --git a/.ahoy.yml b/.ahoy.yml index c7248f98..7d292e4b 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -74,12 +74,12 @@ commands: logs: usage: Show Docker logs. cmd: | + TAIL=100 if [[ $# -eq 2 ]]; then SERVICE=$1 TAIL=$2 else SERVICE=$1 - TAIL=100 fi; docker compose -f $DOCKER_COMPOSE logs -f --tail=$TAIL $SERVICE @@ -124,7 +124,7 @@ commands: usage: Pipe in a postgres dump file. `ahoy db-import local.dump` cmd: | if [ -e "$@" ] ; then - docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v' < "$@" + docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@" else echo "Provided sql file" "$@" "does not exist" fi diff --git a/.env.dbca b/.env.dbca index 0abb4f92..98ba8065 100644 --- a/.env.dbca +++ b/.env.dbca @@ -3,7 +3,6 @@ NGINX_CONTAINER_NAME=nginx REDIS_CONTAINER_NAME=redis POSTGRESQL_CONTAINER_NAME=db SOLR_CONTAINER_NAME=solr -DATAPUSHER_CONTAINER_NAME=datapusher CKAN_CONTAINER_NAME=ckan WORKER_CONTAINER_NAME=ckan-worker diff --git a/.vscode/launch.json b/.vscode/launch.json index 8fd14651..9d24adba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,7 +13,7 @@ "-c continue", "/usr/bin/ckan", "--config", - "/srv/app/ckan.ini", + "/srv/app/config/dbca.ini", "run", "--host", "0.0.0.0", @@ -30,9 +30,24 @@ "-c continue", "/usr/bin/ckan", "--config", - "/srv/app/ckan.ini", + "/srv/app/config/dbca.ini", "jobs", "worker", + "priority" + ], + "justMyCode": true + }, + { + "name": "Python: CKAN CLI Command Run & Debug", + "type": "python", + "request": "launch", + "module": "pdb", + "args": [ + "-c continue", + "/srv/app/config/dbca.ini", + "--config", + "/srv/app/dbca.ini", + "" ], "justMyCode": true } diff --git a/ckan/Dockerfile b/ckan/Dockerfile index f3d0184b..937c3f2a 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -FROM ckan/ckan-base:2.10.1 +FROM ckan/ckan-base:2.10.3 # Install any extensions needed by your CKAN instance # See Dockerfile.dev for more details and examples @@ -21,3 +21,15 @@ RUN for d in $APP_DIR/patches/*; do \ done ; \ fi ; \ done + +## DBCA specific configuration ## + +# Override the default start_ckan.sh script +COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh +RUN chmod +x ${APP_DIR}/start_ckan.sh + +# Copy config files +COPY config/*.ini ${APP_DIR}/config/ + +# Override default CKAN config file to use dbca.ini +ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 9a7403db..bb877c04 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM ckan/ckan-dev:2.10.1 +FROM ckan/ckan-dev:2.10.3 # Install any extensions needed by your CKAN instance # - Make sure to add the plugins to CKAN__PLUGINS in the .env file @@ -38,9 +38,6 @@ RUN pip3 install pip --upgrade && \ chmod +x ${APP_DIR}/dbca_requirements.sh && \ ${APP_DIR}/dbca_requirements.sh -COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh -RUN chmod +x ${APP_DIR}/start_ckan_development.sh - # Clone the extension(s) your are writing for your own project in the `src` folder # to get them mounted in this image at runtime @@ -58,3 +55,15 @@ RUN for d in $APP_DIR/patches/*; do \ done ; \ fi ; \ done + +## DBCA specific configuration ## + +# Override the default dbca_start_ckan_development.sh script +COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh +RUN chmod +x ${APP_DIR}/start_ckan_development.sh + +# Copy config files +COPY config/*.ini ${APP_DIR}/config/ + +# Override default CKAN config file to use dbca.ini +ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini new file mode 100644 index 00000000..ecc26b86 --- /dev/null +++ b/ckan/config/dbca.ini @@ -0,0 +1,64 @@ +# +# DBCA configuration +# + +[app:main] + +## General settings ############################################################ +# This will include the config file settings from the default ckan config file +use = config:/srv/app/ckan.ini +ckan.devserver.watch_patterns = /srv/app/ckan.ini + +## Plugins Settings ############################################################ +ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat envvars + +## Resource Views Settings ##################################################### +ckan.views.default_views = text_view datatables_view pdf_view + +## Internationalisation Settings ############################################### +ckan.locale_default = en_AU +ckan.display_timezone = Australia/West + +## CKAN Extensions configuration ############################################### + + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/ckan/config/uwsgi.ini b/ckan/config/uwsgi.ini new file mode 100644 index 00000000..b60f71e8 --- /dev/null +++ b/ckan/config/uwsgi.ini @@ -0,0 +1,16 @@ +[uwsgi] + +plugins= http,python +socket=/tmp/uwsgi.sock +wsgi-file=/srv/app/wsgi.py +module=wsgi:application +uid=92 +gid=92 +http=0.0.0.0:5000 +master=enable-threads +lazy-apps=true +processes=5 +disable-logging=true +buffer-size=32768 +vacuum=true +harakiri=50 \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override new file mode 100755 index 00000000..174cd06d --- /dev/null +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -0,0 +1,46 @@ +#!/bin/bash + +# Update the default ckan config file +export CKAN_INI=$APP_DIR/ckan.ini + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -qE "beaker.session.secret ?= ?$" ckan.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Run the prerun script to init CKAN and create the default admin user +python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; bash "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + done +fi + +# Revert back touse DBCA config file +export CKAN_INI=$APP_DIR/config/dbca.ini +# Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file +unset CKAN__PLUGINS + +if [ $? -eq 0 ] +then + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + uwsgi -i $APP_DIR/config/uwsgi.ini +else + echo "[prerun] failed...not starting CKAN." +fi \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index 2af71a3e..75cce24e 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -42,7 +42,8 @@ if [ ! -f /tmp/container_ready ]; then fi fi done - + # Update the default ckan config file + export CKAN_INI=$APP_DIR/ckan.ini # Set debug to true echo "Enabling debug mode" ckan config-tool $CKAN_INI -s DEFAULT "debug = true" @@ -88,6 +89,10 @@ if [ ! -f /tmp/container_ready ]; then done fi + # Revert back touse DBCA config file + export CKAN_INI=$APP_DIR/config/dbca.ini + # Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file + unset CKAN__PLUGINS # Set the container as ready so the startup scripts are not run again touch /tmp/container_ready fi diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 521a8112..99a3736a 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -31,6 +31,7 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions + - ./ckan/config:/srv/app/config restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/docker-compose.yml b/docker-compose.yml index 2064ac86..0eaad072 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: networks: - webnet - ckannet + env_file: + - .env depends_on: ckan: condition: service_healthy diff --git a/nginx/Dockerfile b/nginx/Dockerfile index eda7994e..05aeb380 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -9,6 +9,7 @@ RUN apk update --no-cache && \ COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html COPY setup/default.conf ${NGINX_DIR}/conf.d/ +COPY setup/default.conf.template ${NGINX_DIR}/templates/ RUN mkdir -p ${NGINX_DIR}/certs diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template new file mode 100644 index 00000000..fb23b91a --- /dev/null +++ b/nginx/setup/default.conf.template @@ -0,0 +1,46 @@ +server { + #listen 80; + #listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/certs/ckan-local.crt; + ssl_certificate_key /etc/nginx/certs/ckan-local.key; + + # TLS 1.2 & 1.3 only + ssl_protocols TLSv1.2 TLSv1.3; + + # Disable weak ciphers + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + + # SSL sessions + ssl_session_timeout 1d; + # ssl_session_cache dfine in stream and http + ssl_session_tickets off; + + #access_log /var/log/nginx/host.access.log main; + + location / { + proxy_pass http://${CKAN_CONTAINER_NAME:-ckan}:${CKAN_PORT:-5000}/; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $host; + #proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + } + + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; + + # redirect server error pages to the static page /error.html + # + location = /error.html { + ssi on; + internal; + auth_basic off; + root /usr/share/nginx/html; + } + +} \ No newline at end of file From ebe9b6b30186be0b6501b5cc6fe6aa7e5ec2ab1b Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:00:26 +1300 Subject: [PATCH 006/123] Migrated ckan config from .env.dbca to dbca.ini --- .env.dbca | 14 -------------- ckan/config/dbca.ini | 8 ++++++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.env.dbca b/.env.dbca index 98ba8065..df09a41c 100644 --- a/.env.dbca +++ b/.env.dbca @@ -73,23 +73,9 @@ NGINX_PORT=80 NGINX_SSLPORT=443 # Extensions -CKAN__PLUGINS="image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets envvars" -CKAN__VIEWS__DEFAULT_VIEWS="text_view datatables_view pdf_view" -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 ## WA DBCA Config ## # Docker compose project name COMPOSE_PROJECT_NAME=dbca # The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds) DOCKER_COMPOSE=docker-compose.dev.yml - -# Internationalisation Settings -CKAN__DISPLAY_TIMEZONE=Australia/Perth -CKAN__RESOURCE_FORMATS=/srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config/resource_formats.json - -# Scheming -CKAN___SCHEMING__DATASET_SCHEMAS=ckanext.dbca:dbca_dataset.yaml -CKAN___SCHEMING__PRESETS="ckanext.scheming:presets.json ckanext.dbca:dbca_presets.json" diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index ecc26b86..7b0d7724 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,17 +10,21 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat envvars +ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU -ckan.display_timezone = Australia/West +ckan.display_timezone = Australia/Perth +ckan.resource_formats = /srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config/resource_formats.json ## CKAN Extensions configuration ############################################### +# ckanext-scheming +scheming.dataset_schemas = ckanext.dbca:dbca_dataset.yaml +scheming.presets = ckanext.scheming:presets.json ckanext.dbca:dbca_presets.json ## Logging configuration [loggers] From 6e38aba6a0f20a35a774f55718d3ed7dd190cb8f Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:01:01 +1300 Subject: [PATCH 007/123] Updated path to dbca.ini config file --- ckan/setup/dbca_ckan_cron_jobs | 4 ++-- ckan/supervisor/ckan_worker_default.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 3a6bd0e0..442bf281 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,4 +1,4 @@ # Crontab for CKAN cron jobs # Example cron job runs the harvester run command every 15 mins -#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 -0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index f30bf16d..6d8d7303 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -9,7 +9,7 @@ [program:ckan-worker-default] ; Use the full paths to the virtualenv and your configuration file here. -command=/usr/bin/ckan -c /srv/app/ckan.ini jobs worker +command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker ; User the worker runs as. From 39007d95d837ccdde72cb1b81d00e87e462ee9e7 Mon Sep 17 00:00:00 2001 From: awang Date: Wed, 20 Dec 2023 09:53:57 +0700 Subject: [PATCH 008/123] [BIO-6392] removed presets --- ckan/config/dbca.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 7b0d7724..0f88a07c 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -24,7 +24,6 @@ ckan.resource_formats = /srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config # ckanext-scheming scheming.dataset_schemas = ckanext.dbca:dbca_dataset.yaml -scheming.presets = ckanext.scheming:presets.json ckanext.dbca:dbca_presets.json ## Logging configuration [loggers] From b1cd9ebdd9a1d73323a3982c36f7a6308bb4fd24 Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Wed, 27 Dec 2023 09:07:23 +0800 Subject: [PATCH 009/123] [6377] Add spatial extension (#6) * Installed spatial dependencies Enabled spatial plugins Added spatial config value * Enabled dbca plugin --------- Co-authored-by: awang --- ckan/config/dbca.ini | 7 ++++++- ckan/setup/dbca_requirements.sh | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 90798cc3..91dca1ac 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets envvars +ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets spatial_metadata spatial_query envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view @@ -22,6 +22,11 @@ ckan.resource_formats = /srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config ## CKAN Extensions configuration ############################################### +# ckanext-spatial +ckanext.spatial.search_backend = solr-bbox +ckanext.spatial.common_map.custom.url = https://tiles.stadiamaps.com/tiles/stamen_terrain/{z}/{x}/{y}@2x.png +ckanext.spatial.common_map.type = custom + # ckanext-scheming scheming.dataset_schemas = ckanext.dbca:dbca_dataset.yaml diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index 9ffcbe63..db41274c 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -35,6 +35,13 @@ pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming' # Spatial +# dependencies +export PROJ_DIR=/usr +apk add --no-cache \ + geos \ + geos-dev \ + proj-util \ + proj-dev pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt From 32f3f6fe4797d037c6a0fda3357f7d18c1ced365 Mon Sep 17 00:00:00 2001 From: Awang Date: Thu, 11 Jan 2024 08:08:55 +0700 Subject: [PATCH 010/123] [BIO-6368] configured DOI (#7) * [BIO-6368] configured DOI * [BIO-6368] - Updated doi extension to latest tag - Fixed `02_setup-dbca.sh` to get CKAN plugins from the dbca.ini config file and use this file in init db commands - Updated dbca.ini to use new config value `ckanext.doi.language` and revert back to `en_AU` as the ckan locale default --------- Co-authored-by: Mark Calvert --- .env.dbca | 4 ++++ ckan/config/dbca.ini | 8 +++++++- ckan/docker-entrypoint.d/02_setup_dbca.sh | 13 ++++++++++++- ckan/setup/dbca_requirements.sh | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.env.dbca b/.env.dbca index df09a41c..23c45fad 100644 --- a/.env.dbca +++ b/.env.dbca @@ -74,6 +74,10 @@ NGINX_SSLPORT=443 # Extensions +## ckanext-doi ## +CKANEXT__DOI__ACCOUNT_NAME= +CKANEXT__DOI__ACCOUNT_PASSWORD= + ## WA DBCA Config ## # Docker compose project name COMPOSE_PROJECT_NAME=dbca diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 91dca1ac..488196ac 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets spatial_metadata spatial_query envvars +ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets spatial_metadata spatial_query doi envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view @@ -30,6 +30,12 @@ ckanext.spatial.common_map.type = custom # ckanext-scheming scheming.dataset_schemas = ckanext.dbca:dbca_dataset.yaml +# ckanext-doi +ckanext.doi.publisher = Department of Biodiversity, Conservation and Attractions +ckanext.doi.prefix = 10.82800 +ckanext.doi.test_mode = True +ckanext.doi.language = en + ## CKAN Extensions configuration ############################################### ## Logging configuration diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index eb5cb320..2e3f5158 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -1,11 +1,18 @@ #!/bin/bash +# Get the ckan plugins values from the DBCA CKAN config file +CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) +echo "CKAN__PLUGINS: $CKAN__PLUGINS" + if [[ $CKAN__PLUGINS == *"xloader"* ]]; then # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) echo "Setting a temporary value for ckanext.xloader.api_token" ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" fi +# Use the DBCA CKAN config file for the CKAN config file +CKAN_INI=$APP_DIR/config/dbca.ini + ## Examples of how to initialise DB for the extensions # if [[ $CKAN__PLUGINS == *"archiver"* ]]; then # ckan -c $CKAN_INI archiver init @@ -21,4 +28,8 @@ fi if [[ $CKAN__PLUGINS == *"pages"* ]]; then ckan -c $CKAN_INI pages initdb -fi \ No newline at end of file +fi + +if [[ $CKAN__PLUGINS == *"doi"* ]]; then + ckan -c $CKAN_INI doi initdb +fi diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index db41274c..d10db905 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -51,4 +51,4 @@ pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt # 3rd Party # # DOI -pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.9#egg=ckanext-doi \ No newline at end of file +pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi \ No newline at end of file From 192345d269eef8896b663c4944af872b3e774d13 Mon Sep 17 00:00:00 2001 From: Awang Date: Mon, 15 Jan 2024 09:10:37 +0700 Subject: [PATCH 011/123] [BIO-6379] configured geoview ext and re-order plugins (#9) * [BIO-6368] configured DOI * [BIO-6377] switched to solr spatial image * [BIO-6377] added stadia api config * [BIO-6379] configured geoview ext and re-order plugins --------- Co-authored-by: Mark Calvert --- .env.dbca | 5 ++++- ckan/config/dbca.ini | 6 +++--- ckan/setup/dbca_requirements.sh | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.env.dbca b/.env.dbca index 23c45fad..bb4fbf0d 100644 --- a/.env.dbca +++ b/.env.dbca @@ -56,7 +56,7 @@ CKAN_SMTP_MAIL_FROM=ckan@localhost TZ=Australia/Perth # Solr -SOLR_IMAGE_VERSION=2.10-solr9 +SOLR_IMAGE_VERSION=2.10-solr9-spatial CKAN_SOLR_URL=http://solr:8983/solr/ckan TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan @@ -78,6 +78,9 @@ NGINX_SSLPORT=443 CKANEXT__DOI__ACCOUNT_NAME= CKANEXT__DOI__ACCOUNT_PASSWORD= +## ckanext-spatial ## +CKANEXT__SPATIAL__COMMON_MAP__APIKEY= + ## WA DBCA Config ## # Docker compose project name COMPOSE_PROJECT_NAME=dbca diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 488196ac..aa296c1e 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat dbca scheming_datasets spatial_metadata spatial_query doi envvars +ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi resource_proxy geo_view envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = text_view datatables_view pdf_view +ckan.views.default_views = text_view datatables_view pdf_view geo_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU @@ -25,7 +25,7 @@ ckan.resource_formats = /srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config # ckanext-spatial ckanext.spatial.search_backend = solr-bbox ckanext.spatial.common_map.custom.url = https://tiles.stadiamaps.com/tiles/stamen_terrain/{z}/{x}/{y}@2x.png -ckanext.spatial.common_map.type = custom +ckanext.spatial.common_map.type = Stadia.StamenTerrain # ckanext-scheming scheming.dataset_schemas = ckanext.dbca:dbca_dataset.yaml diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index d10db905..fe603b92 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -49,6 +49,9 @@ pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader' pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt +# Geoview +pip3 install -e 'git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckanext-geoview' + # 3rd Party # # DOI -pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi \ No newline at end of file +pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi From ce56f27bb14d00e793f34bfc24b71fc1de52977d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 19 Jan 2024 07:18:15 +1300 Subject: [PATCH 012/123] Added dbca extension --- ckan/setup/dbca_requirements.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index fe603b92..2ffb4799 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -55,3 +55,6 @@ pip3 install -e 'git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckan # 3rd Party # # DOI pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi + +# DBCA Project +pip3 install -e git+https://github.com/dbca-wa/ckanext-dbca.git@develop#egg=ckanext-dbca From 499d40919f36dbd75f5a1b9ab25300d546c4b356 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 09:32:03 +1300 Subject: [PATCH 013/123] Added nginx base image docker-entrypoint.sh to entrypoint so it can run docker-entrypoint.d scripts from base image to use nginx template files. Remove default values for env variables in template file --- nginx/Dockerfile | 2 +- nginx/setup/default.conf.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 05aeb380..2e781263 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -21,4 +21,4 @@ ENTRYPOINT \ -keyout ${NGINX_DIR}/certs/ckan-local.key \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ - nginx -g 'daemon off;' \ No newline at end of file + /docker-entrypoint.sh nginx -g 'daemon off;' \ No newline at end of file diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index fb23b91a..00511652 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -22,7 +22,7 @@ server { #access_log /var/log/nginx/host.access.log main; location / { - proxy_pass http://${CKAN_CONTAINER_NAME:-ckan}:${CKAN_PORT:-5000}/; + proxy_pass http://${CKAN_CONTAINER_NAME}:${CKAN_PORT}/; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header Host $host; #proxy_cache cache; From 5648e171e42df8c97ce2d6c16e00fce43cf08141 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 09:53:57 +1300 Subject: [PATCH 014/123] Set xloader api token in ckan.ini --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 2e3f5158..7176537b 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -5,6 +5,7 @@ CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) echo "CKAN__PLUGINS: $CKAN__PLUGINS" if [[ $CKAN__PLUGINS == *"xloader"* ]]; then + CKAN_INI=$APP_DIR/ckan.ini # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) echo "Setting a temporary value for ckanext.xloader.api_token" ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" From 54d0b0c2a5a014c7a5f30de2fba938dfe7cb295d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 09:58:35 +1300 Subject: [PATCH 015/123] Fixed ahoy command to use correct port from NGINX_SSLPORT_HOST --- .ahoy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ahoy.yml b/.ahoy.yml index 7d292e4b..e56bc399 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -146,7 +146,7 @@ commands: if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then SITE_URL="http://localhost:${CKAN_PORT_HOST}" else - SITE_URL="http://localhost:${NGINX_PORT_HOST}" + SITE_URL="http://localhost:${NGINX_SSLPORT_HOST}" fi open $SITE_URL From d04a304873e7952f61dd729151f2b24a3fcfe06e Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 15:56:00 +1300 Subject: [PATCH 016/123] [BIO-6379] Added shp view plugin and default view --- ckan/config/dbca.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index aa296c1e..982a6ac5 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi resource_proxy geo_view envvars +ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat spatial_metadata spatial_query doi resource_proxy geo_view shp_view envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = text_view datatables_view pdf_view geo_view +ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU @@ -36,7 +36,7 @@ ckanext.doi.prefix = 10.82800 ckanext.doi.test_mode = True ckanext.doi.language = en -## CKAN Extensions configuration ############################################### + ## Logging configuration [loggers] From c3e9741e43d4ffc6c27cdb87edbc7fe3c4bc8580 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 15:57:29 +1300 Subject: [PATCH 017/123] Enabled scheming plugin --- ckan/config/dbca.ini | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 982a6ac5..a800c93c 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat spatial_metadata spatial_query doi resource_proxy geo_view shp_view envvars +ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi resource_proxy geo_view shp_view envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view @@ -36,7 +36,13 @@ ckanext.doi.prefix = 10.82800 ckanext.doi.test_mode = True ckanext.doi.language = en +# ckanext-showcase +ckanext.homepage_views = True +ckanext.showcase.editor = ckeditor +# ckanext-pages +ckanext.pages.allow_html = True +ckanext.pages.editor = ckeditor ## Logging configuration [loggers] From ca5a3318a9fe812197119a59bf2d1cf876eab852 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Jan 2024 16:02:53 +1300 Subject: [PATCH 018/123] Removed won't do extensions --- src/dbca_install_extensions.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index bc6a718e..1ee8b573 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -16,14 +16,5 @@ git clone https://github.com/dbca-wa/ckanext-dbca.git # Office Docs # git clone https://github.com/dbca-wa/ckanext-officedocs -## Should have ## -# Geopusher -# git clone https://github.com/dbca-wa/ckanext-geopusher.git - -## Could have ## -# Cesium Preview -# git clone https://github.com/dbca-wa/ckanext-cesiumpreview.git -# Featured Views -# git clone https://github.com/dbca-wa/ckanext-featuredviews echo "Ready to build project: ahoy build" From b03f0d572efa9bc88788a95560a89f1477f44309 Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:46:23 +1300 Subject: [PATCH 019/123] Feature/6374 qa extension upgrade (#2) * Installed QA extension Enabled plugins archiver & report Added archiver config values Initialise db for archiver and report Added bulk & priority background jobs for archiver Added cron job for report generation Added nginx config to serve archiver cached resources Added ckan_storage volume to nginx to access cached resources * Install qsv dependency for extension ckanext-qa * Simplified installation of qsv * Fixed issues with env variable CKAN__PLUGINS using the value from base image * Added qa dependency file package Updated qa extension to develop branch --- .gitignore | 1 + .vscode/launch.json | 7 ++-- ckan/Dockerfile | 2 +- ckan/Dockerfile.dev | 3 +- ckan/config/dbca.ini | 10 ++++- ckan/docker-entrypoint.d/02_setup_dbca.sh | 29 +++++++------ ckan/setup/dbca_ckan_cron_jobs | 5 ++- ckan/setup/dbca_requirements.sh | 30 +++++++++---- ckan/setup/dbca_start_ckan.sh.override | 4 +- .../dbca_start_ckan_development.sh.override | 10 ++--- ckan/supervisor/ckan_worker_bulk.conf | 42 +++++++++++++++++++ ckan/supervisor/ckan_worker_default.conf | 2 +- ckan/supervisor/ckan_worker_priority.conf | 42 +++++++++++++++++++ docker-compose.yml | 2 + nginx/Dockerfile | 2 +- nginx/setup/default.conf | 4 ++ src/dbca_install_extensions.sh | 11 ++--- 17 files changed, 157 insertions(+), 49 deletions(-) create mode 100644 ckan/supervisor/ckan_worker_bulk.conf create mode 100644 ckan/supervisor/ckan_worker_priority.conf diff --git a/.gitignore b/.gitignore index a3d88d50..b21eec73 100755 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ _solr/schema.xml _src/* local/* .env +dbca/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 9d24adba..faac609b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -32,8 +32,7 @@ "--config", "/srv/app/config/dbca.ini", "jobs", - "worker", - "priority" + "worker" ], "justMyCode": true }, @@ -44,9 +43,9 @@ "module": "pdb", "args": [ "-c continue", - "/srv/app/config/dbca.ini", + "/usr/bin/ckan", "--config", - "/srv/app/dbca.ini", + "/srv/app/config/dbca.ini", "" ], "justMyCode": true diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 937c3f2a..1b0176da 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -32,4 +32,4 @@ RUN chmod +x ${APP_DIR}/start_ckan.sh COPY config/*.ini ${APP_DIR}/config/ # Override default CKAN config file to use dbca.ini -ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file +ENV CKAN_INI=${APP_DIR}/config/dbca.ini diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index bb877c04..0fb294fa 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -31,7 +31,6 @@ FROM ckan/ckan-dev:2.10.3 #RUN pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ # pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt - # Install any extensions needed by your CKAN instance COPY setup/dbca_requirements.sh ${APP_DIR} RUN pip3 install pip --upgrade && \ @@ -66,4 +65,4 @@ RUN chmod +x ${APP_DIR}/start_ckan_development.sh COPY config/*.ini ${APP_DIR}/config/ # Override default CKAN config file to use dbca.ini -ENV CKAN_INI=${APP_DIR}/config/dbca.ini \ No newline at end of file +ENV CKAN_INI=${APP_DIR}/config/dbca.ini diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index a800c93c..b6e6174f 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi resource_proxy geo_view shp_view envvars +ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view @@ -44,6 +44,14 @@ ckanext.showcase.editor = ckeditor ckanext.pages.allow_html = True ckanext.pages.editor = ckeditor +# ckanext-archiver +ckanext-archiver.archive_dir=/var/lib/ckan/archiver +ckanext-archiver.cache_url_root=/resource_cache/ + +# ckanext-qa +ckanext.qa.qsv_bin=/usr/local/bin/qsv + + ## Logging configuration [loggers] keys = root, ckan, ckanext, werkzeug diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 7176537b..f4741616 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -1,7 +1,5 @@ #!/bin/bash -# Get the ckan plugins values from the DBCA CKAN config file -CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) echo "CKAN__PLUGINS: $CKAN__PLUGINS" if [[ $CKAN__PLUGINS == *"xloader"* ]]; then @@ -10,22 +8,19 @@ if [[ $CKAN__PLUGINS == *"xloader"* ]]; then echo "Setting a temporary value for ckanext.xloader.api_token" ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" fi +CKAN_INI=$APP_DIR/config/dbca.ini -# Use the DBCA CKAN config file for the CKAN config file -CKAN_INI=$APP_DIR/config/dbca.ini - -## Examples of how to initialise DB for the extensions -# if [[ $CKAN__PLUGINS == *"archiver"* ]]; then -# ckan -c $CKAN_INI archiver init -# fi +if [[ $CKAN__PLUGINS == *"archiver"* ]]; then + ckan -c $CKAN_INI archiver init +fi -# if [[ $CKAN__PLUGINS == *"report"* ]]; then -# ckan -c $CKAN_INI report initdb -# fi +if [[ $CKAN__PLUGINS == *"report"* ]]; then + ckan -c $CKAN_INI report initdb +fi -# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then -# ckan -c $CKAN_INI db upgrade -p harvest -# fi +if [[ $CKAN__PLUGINS == *"qa"* ]]; then + ckan -c $CKAN_INI qa init +fi if [[ $CKAN__PLUGINS == *"pages"* ]]; then ckan -c $CKAN_INI pages initdb @@ -34,3 +29,7 @@ fi if [[ $CKAN__PLUGINS == *"doi"* ]]; then ckan -c $CKAN_INI doi initdb fi + +# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then +# ckan -c $CKAN_INI db upgrade -p harvest +# fi \ No newline at end of file diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 442bf281..5c9442ff 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,4 +1,7 @@ # Crontab for CKAN cron jobs # Example cron job runs the harvester run command every 15 mins -#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +# Midnight task to schedule embargo datasets to public visibility 0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +# Midnight report generation for archiver broken links +0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini report generate >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index 2ffb4799..eb0d7e81 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -1,8 +1,9 @@ #!/bin/sh -## Must Have ## +## CKAN Core extensions ## + # Archiver -pip3 install -e 'git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver' +pip3 install -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver pip3 install -r ${SRC_DIR}/ckanext-archiver/requirements.txt # DCAT @@ -10,7 +11,7 @@ pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext- pip3 install -r ${SRC_DIR}/ckanext-dcat/requirements.txt # Harvester -pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest' +pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest pip3 install -r ${SRC_DIR}/ckanext-harvest/requirements.txt # Hierarchy @@ -32,7 +33,7 @@ pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.6.1#egg=ckan pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt # Scheming -pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming' +pip3 install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming # Spatial # dependencies @@ -46,15 +47,28 @@ pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckane pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt # XLoader -pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader' +pip3 install -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt # Geoview -pip3 install -e 'git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckanext-geoview' +pip3 install -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckanext-geoview + -# 3rd Party # +## 3rd Party ## # DOI pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi -# DBCA Project + +## DBCA Project ## + +# DBCA pip3 install -e git+https://github.com/dbca-wa/ckanext-dbca.git@develop#egg=ckanext-dbca + +# QA +# Install qsv dependency for extension ckanext-qa +wget -O /tmp/qsv.zip https://github.com/jqnatividad/qsv/releases/download/0.110.0/qsv-0.110.0-x86_64-unknown-linux-musl.zip +unzip /tmp/qsv.zip -d /usr/local/bin +rm /tmp/qsv.zip +apk add file +pip3 install -e git+https://github.com/dbca-wa/ckanext-qa.git@develop#egg=ckanext-qa +pip3 install -r ${SRC_DIR}/ckanext-qa/requirements.txt diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index 174cd06d..f4b19028 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -1,5 +1,7 @@ #!/bin/bash +# Set the CKAN plugins variable to the plugins value from the dbca.ini config file +export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) # Update the default ckan config file export CKAN_INI=$APP_DIR/ckan.ini @@ -32,8 +34,6 @@ fi # Revert back touse DBCA config file export CKAN_INI=$APP_DIR/config/dbca.ini -# Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file -unset CKAN__PLUGINS if [ $? -eq 0 ] then diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index 75cce24e..d5fdafeb 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -1,5 +1,8 @@ #!/bin/sh +# Set the CKAN plugins variable to the plugins value from the dbca.ini config file +export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) + # Only run these start up scripts the first time the container is created if [ ! -f /tmp/container_ready ]; then # Install any local extensions in the src_extensions volume @@ -60,10 +63,6 @@ if [ ! -f /tmp/container_ready ]; then ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" fi - # Update the plugins setting in the ini file with the values defined in the env var - echo "Loading the following plugins: $CKAN__PLUGINS" - ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" - # Update test-core.ini DB, SOLR & Redis settings echo "Loading test settings into test-core.ini" ckan config-tool $SRC_DIR/ckan/test-core.ini \ @@ -91,8 +90,7 @@ if [ ! -f /tmp/container_ready ]; then # Revert back touse DBCA config file export CKAN_INI=$APP_DIR/config/dbca.ini - # Unset CKAN__PLUGINS to stop the extension envvars overriding the plugins value set in the dbca config file - unset CKAN__PLUGINS + # Set the container as ready so the startup scripts are not run again touch /tmp/container_ready fi diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf new file mode 100644 index 00000000..1aaa2ed4 --- /dev/null +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -0,0 +1,42 @@ +; ======================================================= +; Supervisor configuration for CKAN background job worker +; ======================================================= + +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup + + +[program:ckan-worker-bulk] + +; Use the full paths to the virtualenv and your configuration file here. +command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk" + + +; User the worker runs as. +user=ckan + + +; Start just a single worker. Increase this number if you have many or +; particularly long running background jobs. +numprocs=1 +process_name=%(program_name)s-%(process_num)02d + + +; Log files. +stdout_logfile=/srv/app/logs/ckan-worker.stdout.log +stderr_logfile=/srv/app/logs/ckan-worker.stderr.log + + +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true + + +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 + +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index 6d8d7303..30378a35 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -9,7 +9,7 @@ [program:ckan-worker-default] ; Use the full paths to the virtualenv and your configuration file here. -command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker +command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker" ; User the worker runs as. diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf new file mode 100644 index 00000000..bd348274 --- /dev/null +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -0,0 +1,42 @@ +; ======================================================= +; Supervisor configuration for CKAN background job worker +; ======================================================= + +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup + + +[program:ckan-worker-priority] + +; Use the full paths to the virtualenv and your configuration file here. +command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority" + + +; User the worker runs as. +user=ckan + + +; Start just a single worker. Increase this number if you have many or +; particularly long running background jobs. +numprocs=1 +process_name=%(program_name)s-%(process_num)02d + + +; Log files. +stdout_logfile=/srv/app/logs/ckan-worker.stdout.log +stderr_logfile=/srv/app/logs/ckan-worker.stderr.log + + +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true + + +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 + +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 diff --git a/docker-compose.yml b/docker-compose.yml index 0eaad072..29d12c04 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: condition: service_healthy ports: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" + volumes: + - ckan_storage:/var/lib/ckan ckan: container_name: ${CKAN_CONTAINER_NAME} diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 2e781263..8e6bbce5 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -21,4 +21,4 @@ ENTRYPOINT \ -keyout ${NGINX_DIR}/certs/ckan-local.key \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ - /docker-entrypoint.sh nginx -g 'daemon off;' \ No newline at end of file + /docker-entrypoint.sh nginx -g 'daemon off;' diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index a628619f..c93e5ccd 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -32,6 +32,10 @@ server { proxy_cache_key $host$scheme$proxy_host$request_uri; } + # archived files from ckanext-archiver + location /resource_cache/ { + alias /var/lib/ckan/archiver/; + } error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; # redirect server error pages to the static page /error.html diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index 1ee8b573..a4db0509 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -5,16 +5,13 @@ # Uncomment the following lines to install these extension you are working on to upgrade to CKAN 2.10 cd src/ - -## Must Have ## +# DBCA git clone https://github.com/dbca-wa/ckanext-dbca.git - # QA -# git clone https://github.com/dbca-wa/ckanext-qa.git -# These extensions will be installed by default, but we don't want them -# sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt +git clone https://github.com/dbca-wa/ckanext-qa.git +#These extensions will be installed by default, but we don't want them +sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt # Office Docs # git clone https://github.com/dbca-wa/ckanext-officedocs - echo "Ready to build project: ahoy build" From e232ce297de1cc49177ee1bb3d4b542cb484fb7f Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 1 Feb 2024 11:40:23 +1300 Subject: [PATCH 020/123] Updated project to use log files for ckan, ckan-worker, ckan-cron-jobs, nginx --- ckan/Dockerfile | 6 ++++++ ckan/Dockerfile.dev | 6 ++++++ ckan/Dockerfile.worker | 6 ------ ckan/config/dbca.ini | 16 +++++++++++----- ckan/setup/dbca_ckan_cron_jobs | 6 +++--- ckan/supervisor/ckan_cron_jobs.conf | 8 +++++--- ckan/supervisor/ckan_worker_bulk.conf | 6 ++++-- ckan/supervisor/ckan_worker_default.conf | 6 ++++-- ckan/supervisor/ckan_worker_priority.conf | 6 ++++-- docker-compose.dev.yml | 4 ++++ docker-compose.yml | 4 ++++ nginx/setup/default.conf | 5 ++++- nginx/setup/default.conf.template | 5 ++++- 13 files changed, 59 insertions(+), 25 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 1b0176da..46cf8c80 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -24,6 +24,12 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +## Create logs folder +RUN mkdir -p $APP_DIR/logs && \ + touch $APP_DIR/logs/ckan.log && \ + chown -R ckan:ckan $APP_DIR/logs && \ + chmod -R 755 $APP_DIR/logs + # Override the default start_ckan.sh script COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh RUN chmod +x ${APP_DIR}/start_ckan.sh diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 0fb294fa..aa59b66c 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,6 +57,12 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +## Create logs folder +RUN mkdir -p $APP_DIR/logs && \ + touch $APP_DIR/logs/ckan.log && \ + chown -R ckan:ckan $APP_DIR/logs && \ + chmod -R 755 $APP_DIR/logs + # Override the default dbca_start_ckan_development.sh script COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh RUN chmod +x ${APP_DIR}/start_ckan_development.sh diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 4c8a4657..03853739 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -1,12 +1,6 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} -## Create logs folder -RUN mkdir -p $APP_DIR/logs && \ - touch $APP_DIR/logs/ckan-cron-jobs.log && \ - chown -R ckan:ckan $APP_DIR/logs && \ - chmod -R 755 $APP_DIR/logs - ## Supervisor config COPY supervisor/*.conf /etc/supervisord.d diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index b6e6174f..2f423bb1 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -57,30 +57,30 @@ ckanext.qa.qsv_bin=/usr/local/bin/qsv keys = root, ckan, ckanext, werkzeug [handlers] -keys = console +keys = console, fileHandler [formatters] keys = generic [logger_root] level = WARNING -handlers = console +handlers = console, fileHandler [logger_werkzeug] level = WARNING -handlers = console +handlers = console, fileHandler qualname = werkzeug propagate = 0 [logger_ckan] level = INFO -handlers = console +handlers = console, fileHandler qualname = ckan propagate = 0 [logger_ckanext] level = DEBUG -handlers = console +handlers = console, fileHandler qualname = ckanext propagate = 0 @@ -90,5 +90,11 @@ args = (sys.stderr,) level = NOTSET formatter = generic +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +args = ('/srv/app/logs/ckan.log', 'midnight', 1, 0) +level = NOTSET +formatter = generic + [formatter_generic] format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 5c9442ff..618ccd81 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,7 +1,7 @@ # Crontab for CKAN cron jobs # Example cron job runs the harvester run command every 15 mins -#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # Midnight task to schedule embargo datasets to public visibility -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets # Midnight report generation for archiver broken links -0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini report generate >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate \ No newline at end of file diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index d3268a40..b89069ba 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -9,7 +9,10 @@ [program:ckan-cron-jobs] ; Use the full paths to the virtualenv and your configuration file here. -command=/usr/sbin/crond -f -L /srv/app/logs/ckan-cron-jobs.log +# This configuration sets up a cron job for CKAN. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini file. +# It then executes the crond command with the -f flag to run in the foreground. +command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/sbin/crond -f" ; User the worker runs as. @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-crons-jobs.stdout.log -stderr_logfile=/srv/app/logs/ckan-crons-jobs.stderr.log +stderr_logfile=/srv/app/logs/ckan-crons-jobs.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf index 1aaa2ed4..02385fcf 100644 --- a/ckan/supervisor/ckan_worker_bulk.conf +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -9,6 +9,9 @@ [program:ckan-worker-bulk] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker bulk command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index 30378a35..b3ac1861 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -9,6 +9,9 @@ [program:ckan-worker-default] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf index bd348274..a7fc1d43 100644 --- a/ckan/supervisor/ckan_worker_priority.conf +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -9,6 +9,9 @@ [program:ckan-worker-priority] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker priority command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 99a3736a..496ede1c 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,6 +4,7 @@ volumes: ckan_storage: pg_data: solr_data: + ckan_logs: services: @@ -32,6 +33,7 @@ services: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions - ./ckan/config:/srv/app/config + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] @@ -62,6 +64,8 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions + - ./ckan/config:/srv/app/config + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/docker-compose.yml b/docker-compose.yml index 29d12c04..b9be683b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ volumes: ckan_storage: pg_data: solr_data: + ckan_logs: services: @@ -25,6 +26,7 @@ services: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs ckan: container_name: ${CKAN_CONTAINER_NAME} @@ -53,6 +55,7 @@ services: condition: service_healthy volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] @@ -87,6 +90,7 @@ services: condition: service_started volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index c93e5ccd..de344696 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -19,7 +19,10 @@ server { # ssl_session_cache dfine in stream and http ssl_session_tickets off; - #access_log /var/log/nginx/host.access.log main; + access_log /srv/app/logs/nginx_access.log main; + access_log /var/log/nginx/access.log main; + error_log /srv/app/logs/nginx_error.log error; + error_log /var/log/nginx/errror.log error; location / { proxy_pass http://ckan:5000/; diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index 00511652..e57fbb50 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -19,7 +19,10 @@ server { # ssl_session_cache dfine in stream and http ssl_session_tickets off; - #access_log /var/log/nginx/host.access.log main; + access_log /srv/app/logs/nginx_access.log main; + access_log /var/log/nginx/access.log main; + error_log /srv/app/logs/nginx_error.log error; + error_log /var/log/nginx/errror.log error; location / { proxy_pass http://${CKAN_CONTAINER_NAME}:${CKAN_PORT}/; From 2610e600f849d86eb16d338fa52142c56682615c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 1 Feb 2024 12:18:05 +1300 Subject: [PATCH 021/123] Update ckan.resource_formats path in dbca.ini --- ckan/config/dbca.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 2f423bb1..4a3fd3d3 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -18,7 +18,7 @@ ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU ckan.display_timezone = Australia/Perth -ckan.resource_formats = /srv/app/src_extensions/ckanext-dbca/ckanext/dbca/config/resource_formats.json +ckan.resource_formats = /srv/app/src/ckanext-dbca/ckanext/dbca/config/resource_formats.json ## CKAN Extensions configuration ############################################### From 40500ba17debc4e964a61cac9343f2216ee0cdbd Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 1 Feb 2024 15:49:51 +1300 Subject: [PATCH 022/123] Add resource_cache location in nginx configuration template --- nginx/setup/default.conf | 1 + nginx/setup/default.conf.template | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index de344696..d6419dbc 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -39,6 +39,7 @@ server { location /resource_cache/ { alias /var/lib/ckan/archiver/; } + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; # redirect server error pages to the static page /error.html diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index e57fbb50..0e0da7b3 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -35,6 +35,11 @@ server { proxy_cache_key $host$scheme$proxy_host$request_uri; } + # archived files from ckanext-archiver + location /resource_cache/ { + alias /var/lib/ckan/archiver/; + } + error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 421 422 423 424 425 426 428 429 431 451 500 501 502 503 504 505 506 507 508 510 511 /error.html; # redirect server error pages to the static page /error.html From 57c8ed2fd6d20fe106ff0e7eeb8b0211b749dc45 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 1 Feb 2024 16:55:46 +1300 Subject: [PATCH 023/123] Installed and configured extension ckanext-officedocs --- .env.dbca | 5 ++++- ckan/config/dbca.ini | 4 ++-- ckan/setup/dbca_requirements.sh | 3 +++ src/dbca_install_extensions.sh | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.env.dbca b/.env.dbca index bb4fbf0d..19df1b54 100644 --- a/.env.dbca +++ b/.env.dbca @@ -37,7 +37,10 @@ USE_HTTPS_FOR_DEV=false # CKAN core CKAN_VERSION=2.10.0 CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 +# CKAN Dev +CKAN_SITE_URL=http://localhost:$CKAN_PORT_HOST +# CKAN Nginx +#CKAN_SITE_URL=https://localhost:$NGINX_SSLPORT_HOST CKAN_PORT=5000 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 4a3fd3d3..cd0022f0 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view envvars +ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view +ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view officedocs_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index eb0d7e81..b5847a69 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -72,3 +72,6 @@ rm /tmp/qsv.zip apk add file pip3 install -e git+https://github.com/dbca-wa/ckanext-qa.git@develop#egg=ckanext-qa pip3 install -r ${SRC_DIR}/ckanext-qa/requirements.txt + +# Office Docs +pip3 install -e git+https://github.com/dbca-wa/ckanext-officedocs.git@develop#egg=ckanext-officedocs \ No newline at end of file diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index a4db0509..1d21f24e 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -12,6 +12,6 @@ git clone https://github.com/dbca-wa/ckanext-qa.git #These extensions will be installed by default, but we don't want them sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt # Office Docs -# git clone https://github.com/dbca-wa/ckanext-officedocs +git clone https://github.com/dbca-wa/ckanext-officedocs.git echo "Ready to build project: ahoy build" From 3d3cdd33166f923321d7d31e4c3b01bfc25bc04f Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 7 Feb 2024 10:58:50 +1300 Subject: [PATCH 024/123] Added ahoy command and env variables to download CKAN logs --- .ahoy.yml | 29 +++++++++++++++++++++++++++++ .env.dbca | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/.ahoy.yml b/.ahoy.yml index e56bc399..4a7f066a 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -150,3 +150,32 @@ commands: fi open $SITE_URL + download-ckan-logs: + usage: Download CKAN logs + cmd: | + if [ -z "$AZURE_FILE_SHARE_URL" ] || [ -z "$AZURE_FILE_SHARE_SAS_TOKEN" ]; then + echo "AZURE_FILE_SHARE_URL or AZURE_FILE_SHARE_SAS_TOKEN is not set" + exit 1 + fi + echo "Downloading ckan_logs from:$AZURE_FILE_SHARE_URL" + LOG_FILE="ckan.log" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + echo "Downloading ckan.log:$FULL_URL" + curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + echo "Downloading ckan-crons-jobs.log" + LOG_FILE="ckan-crons-jobs.log" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + echo "Downloading ckan-worker.log" + LOG_FILE="ckan-worker.log" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + echo "Downloading nginx access log" + LOG_FILE="nginx_access.log" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + echo "Downloading nginx error log" + LOG_FILE="nginx_access.log" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + echo "CKAN logs downloaded to dbca/ckan_logs" \ No newline at end of file diff --git a/.env.dbca b/.env.dbca index 19df1b54..f39efa2f 100644 --- a/.env.dbca +++ b/.env.dbca @@ -89,3 +89,7 @@ CKANEXT__SPATIAL__COMMON_MAP__APIKEY= COMPOSE_PROJECT_NAME=dbca # The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds) DOCKER_COMPOSE=docker-compose.dev.yml + +# Azure File Share Storage +AZURE_FILE_SHARE_URL= +AZURE_FILE_SHARE_SAS_TOKEN= From afff78aaf38f9ead2ae36265169acc6f7d05d94a Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 7 Feb 2024 12:01:46 +1300 Subject: [PATCH 025/123] Create additional CKAN log files in Dockerfile --- ckan/Dockerfile | 4 +++- ckan/Dockerfile.dev | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 46cf8c80..192101cc 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -24,9 +24,11 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## -## Create logs folder +## Create logs folder and files RUN mkdir -p $APP_DIR/logs && \ touch $APP_DIR/logs/ckan.log && \ + touch $APP_DIR/logs/ckan-worker.log && \ + touch $APP_DIR/logs/ckan-crons-jobs.log && \ chown -R ckan:ckan $APP_DIR/logs && \ chmod -R 755 $APP_DIR/logs diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index aa59b66c..2696ee93 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,9 +57,11 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## -## Create logs folder +## Create logs folder and files RUN mkdir -p $APP_DIR/logs && \ touch $APP_DIR/logs/ckan.log && \ + touch $APP_DIR/logs/ckan-worker.log && \ + touch $APP_DIR/logs/ckan-crons-jobs.log && \ chown -R ckan:ckan $APP_DIR/logs && \ chmod -R 755 $APP_DIR/logs From 6b32b29fac7bb90910f43c8f8db42d1fd37b7155 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 7 Feb 2024 15:27:16 +1300 Subject: [PATCH 026/123] Add logrotate for CKAN app & Nginx logs --- ckan/Dockerfile.worker | 5 +++++ ckan/setup/dbca_ckan_cron_jobs | 2 ++ ckan/setup/dbca_ckan_rotate_logs | 9 +++++++++ nginx/Dockerfile | 8 ++++++++ nginx/setup/dbca_nginx_cron_jobs | 3 +++ nginx/setup/dbca_nginx_rotate_logs | 9 +++++++++ 6 files changed, 36 insertions(+) create mode 100644 ckan/setup/dbca_ckan_rotate_logs create mode 100644 nginx/setup/dbca_nginx_cron_jobs create mode 100644 nginx/setup/dbca_nginx_rotate_logs diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 03853739..0e57d586 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -1,6 +1,11 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} +# Install logrotate +RUN apk add logrotate +# Create logrotate config file for app logs +COPY setup/dbca_ckan_rotate_logs /etc/logrotate.d/dbca_ckan_rotate_logs + ## Supervisor config COPY supervisor/*.conf /etc/supervisord.d diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 618ccd81..cc3ea815 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,4 +1,6 @@ # Crontab for CKAN cron jobs +# Midnight task to rotate CKAN app logs every day +0 0 * * * /usr/sbin/logrotate -s /srv/app/logs/logrotate.status /etc/logrotate.d/dbca_ckan_rotate_logs # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # Midnight task to schedule embargo datasets to public visibility diff --git a/ckan/setup/dbca_ckan_rotate_logs b/ckan/setup/dbca_ckan_rotate_logs new file mode 100644 index 00000000..71161c14 --- /dev/null +++ b/ckan/setup/dbca_ckan_rotate_logs @@ -0,0 +1,9 @@ +/srv/app/logs/ckan-worker.log /srv/app/logs/ckan-crons-jobs.log { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + copytruncate +} \ No newline at end of file diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 8e6bbce5..aed247e3 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -13,6 +13,13 @@ COPY setup/default.conf.template ${NGINX_DIR}/templates/ RUN mkdir -p ${NGINX_DIR}/certs +# Install logrotate +RUN apk add logrotate +# Create logrotate config file for nginx logs +COPY setup/dbca_nginx_rotate_logs /etc/logrotate.d/dbca_nginx_rotate_logs +## Cron jobs config +COPY setup/dbca_nginx_cron_jobs /etc/crontabs/root + ENTRYPOINT \ openssl req \ -subj '/C=DE/ST=Berlin/L=Berlin/O=None/CN=localhost' \ @@ -21,4 +28,5 @@ ENTRYPOINT \ -keyout ${NGINX_DIR}/certs/ckan-local.key \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ + /usr/sbin/crond && \ /docker-entrypoint.sh nginx -g 'daemon off;' diff --git a/nginx/setup/dbca_nginx_cron_jobs b/nginx/setup/dbca_nginx_cron_jobs new file mode 100644 index 00000000..31921db7 --- /dev/null +++ b/nginx/setup/dbca_nginx_cron_jobs @@ -0,0 +1,3 @@ +# Crontab for Nginx cron jobs +# Midnight task to rotate CKAN app logs every day +0 0 * * * /usr/sbin/logrotate -s /srv/app/logs/logrotate.status /etc/logrotate.d/dbca_nginx_rotate_logs diff --git a/nginx/setup/dbca_nginx_rotate_logs b/nginx/setup/dbca_nginx_rotate_logs new file mode 100644 index 00000000..73145028 --- /dev/null +++ b/nginx/setup/dbca_nginx_rotate_logs @@ -0,0 +1,9 @@ +/srv/app/logs/nginx_access.log /srv/app/logs/nginx_error.log { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + copytruncate +} \ No newline at end of file From 7b3f4aa1e87769384dced2f6b8fc961025396345 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 8 Feb 2024 11:43:03 +1300 Subject: [PATCH 027/123] Fixed issues with downloading CKAN logs from Azure file share --- .ahoy.yml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index 4a7f066a..23f2621c 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -157,25 +157,32 @@ commands: echo "AZURE_FILE_SHARE_URL or AZURE_FILE_SHARE_SAS_TOKEN is not set" exit 1 fi + echo "Downloading ckan_logs from:$AZURE_FILE_SHARE_URL" + LOG_FILE="ckan.log" + echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - echo "Downloading ckan.log:$FULL_URL" - curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" - echo "Downloading ckan-crons-jobs.log" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + LOG_FILE="ckan-crons-jobs.log" + echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" - echo "Downloading ckan-worker.log" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + LOG_FILE="ckan-worker.log" + echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" - echo "Downloading nginx access log" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + LOG_FILE="nginx_access.log" + echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" - echo "Downloading nginx error log" - LOG_FILE="nginx_access.log" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + + LOG_FILE="nginx_error.log" + echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - curl -L -o dbca/ckan_logs/ckan.log "$FULL_URL" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + echo "CKAN logs downloaded to dbca/ckan_logs" \ No newline at end of file From d83eb8b8d0a681a9a37d1051875acb55a0b611f2 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 8 Feb 2024 12:24:01 +1300 Subject: [PATCH 028/123] Refactor Dockerfile and setup script to create logs folder and files --- ckan/Dockerfile | 8 -------- ckan/Dockerfile.dev | 8 -------- ckan/docker-entrypoint.d/02_setup_dbca.sh | 7 +++++++ 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 192101cc..1b0176da 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -24,14 +24,6 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## -## Create logs folder and files -RUN mkdir -p $APP_DIR/logs && \ - touch $APP_DIR/logs/ckan.log && \ - touch $APP_DIR/logs/ckan-worker.log && \ - touch $APP_DIR/logs/ckan-crons-jobs.log && \ - chown -R ckan:ckan $APP_DIR/logs && \ - chmod -R 755 $APP_DIR/logs - # Override the default start_ckan.sh script COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh RUN chmod +x ${APP_DIR}/start_ckan.sh diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 2696ee93..0fb294fa 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,14 +57,6 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## -## Create logs folder and files -RUN mkdir -p $APP_DIR/logs && \ - touch $APP_DIR/logs/ckan.log && \ - touch $APP_DIR/logs/ckan-worker.log && \ - touch $APP_DIR/logs/ckan-crons-jobs.log && \ - chown -R ckan:ckan $APP_DIR/logs && \ - chmod -R 755 $APP_DIR/logs - # Override the default dbca_start_ckan_development.sh script COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh RUN chmod +x ${APP_DIR}/start_ckan_development.sh diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index f4741616..c5bbf663 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -1,5 +1,12 @@ #!/bin/bash +## Create logs folder and files +mkdir -p $APP_DIR/logs +touch $APP_DIR/logs/ckan.log +touch $APP_DIR/logs/ckan-worker.log +touch $APP_DIR/logs/ckan-crons-jobs.log +chown -R ckan:ckan $APP_DIR/logs + echo "CKAN__PLUGINS: $CKAN__PLUGINS" if [[ $CKAN__PLUGINS == *"xloader"* ]]; then From b345d9b9d0e3461d40003f3d9ddbc87fe7d88ce9 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 8 Feb 2024 13:50:10 +1300 Subject: [PATCH 029/123] Add client_max_body_size directive to nginx configuration --- nginx/setup/default.conf | 2 ++ nginx/setup/default.conf.template | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index d6419dbc..61359b65 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -24,6 +24,8 @@ server { error_log /srv/app/logs/nginx_error.log error; error_log /var/log/nginx/errror.log error; + client_max_body_size 100M; + location / { proxy_pass http://ckan:5000/; proxy_set_header X-Forwarded-For $remote_addr; diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index 0e0da7b3..ca80d5bf 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -24,6 +24,8 @@ server { error_log /srv/app/logs/nginx_error.log error; error_log /var/log/nginx/errror.log error; + client_max_body_size 100M; + location / { proxy_pass http://${CKAN_CONTAINER_NAME}:${CKAN_PORT}/; proxy_set_header X-Forwarded-For $remote_addr; From 6cc5abe003dec8f2003ac3ced4096e4d474b90f4 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 8 Feb 2024 10:41:54 +0800 Subject: [PATCH 030/123] Add archive folder and set permissions Update log rotation to use ckan user --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 6 +++++- ckan/setup/dbca_ckan_rotate_logs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index c5bbf663..c2564cb3 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -1,12 +1,16 @@ #!/bin/bash -## Create logs folder and files +## Create logs folder/files and set permissions mkdir -p $APP_DIR/logs touch $APP_DIR/logs/ckan.log touch $APP_DIR/logs/ckan-worker.log touch $APP_DIR/logs/ckan-crons-jobs.log chown -R ckan:ckan $APP_DIR/logs +## Create archive folder and set permissions +mkdir -p $CKAN_STORAGE_PATH/archiver +chown -R ckan:ckan $CKAN_STORAGE_PATH/archiver + echo "CKAN__PLUGINS: $CKAN__PLUGINS" if [[ $CKAN__PLUGINS == *"xloader"* ]]; then diff --git a/ckan/setup/dbca_ckan_rotate_logs b/ckan/setup/dbca_ckan_rotate_logs index 71161c14..f1af6a96 100644 --- a/ckan/setup/dbca_ckan_rotate_logs +++ b/ckan/setup/dbca_ckan_rotate_logs @@ -1,4 +1,5 @@ /srv/app/logs/ckan-worker.log /srv/app/logs/ckan-crons-jobs.log { + su ckan ckan daily rotate 7 missingok From c0123dbc977e4db4f25dc0b938c07880090ecba3 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 8 Feb 2024 17:17:15 +1300 Subject: [PATCH 031/123] Update CKAN worker configuration --- ckan/Dockerfile.worker | 4 ++++ ckan/supervisor/ckan_cron_jobs.conf | 4 +--- ckan/supervisor/ckan_worker_bulk.conf | 4 +--- ckan/supervisor/ckan_worker_default.conf | 4 +--- ckan/supervisor/ckan_worker_priority.conf | 4 +--- docker-compose.dev.yml | 1 - 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 0e57d586..5a988415 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -1,6 +1,10 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} +# Override the default CKAN config plugins to only use the ones we need for the worker +ENV CKAN__PLUGINS="dbca datastore xloader scheming_datasets qa archiver report envvars" +RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" + # Install logrotate RUN apk add logrotate # Create logrotate config file for app logs diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index b89069ba..ed5e3fa9 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -10,9 +10,7 @@ ; Use the full paths to the virtualenv and your configuration file here. # This configuration sets up a cron job for CKAN. -# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini file. -# It then executes the crond command with the -f flag to run in the foreground. -command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/sbin/crond -f" +command= /usr/sbin/crond -f ; User the worker runs as. diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf index 02385fcf..b79065c1 100644 --- a/ckan/supervisor/ckan_worker_bulk.conf +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -10,9 +10,7 @@ ; Use the full paths to the virtualenv and your configuration file here. # This configuration file specifies the command to run for the CKAN worker bulk process. -# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. -# It then executes the ckan command with the specified configuration file and runs the jobs worker bulk command. -command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk" +command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk ; User the worker runs as. diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index b3ac1861..354bfc1b 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -10,9 +10,7 @@ ; Use the full paths to the virtualenv and your configuration file here. # This configuration file specifies the command to run for the CKAN worker bulk process. -# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. -# It then executes the ckan command with the specified configuration file and runs the jobs worker command. -command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker" +command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker ; User the worker runs as. diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf index a7fc1d43..195ad982 100644 --- a/ckan/supervisor/ckan_worker_priority.conf +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -10,9 +10,7 @@ ; Use the full paths to the virtualenv and your configuration file here. # This configuration file specifies the command to run for the CKAN worker bulk process. -# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. -# It then executes the ckan command with the specified configuration file and runs the jobs worker priority command. -command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority" +command=/usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority ; User the worker runs as. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 496ede1c..86be507b 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -64,7 +64,6 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions - - ./ckan/config:/srv/app/config - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: From 8882f7b4b63e3d2e75364d27f2a70b7f094902ff Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 14 Feb 2024 10:10:33 +0800 Subject: [PATCH 032/123] Update CKAN Dockerfile.worker with CKAN_SITE_URL environment variable to use the localhost CKAN Web server for CKAN job workers --- ckan/Dockerfile.worker | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 5a988415..92a40a93 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -4,6 +4,7 @@ FROM ${CKAN_IMAGE} # Override the default CKAN config plugins to only use the ones we need for the worker ENV CKAN__PLUGINS="dbca datastore xloader scheming_datasets qa archiver report envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" +ENV CKAN_SITE_URL=http://localhost:5000 # Install logrotate RUN apk add logrotate From 9af425901d1d0f966b92b6bce6151d8d18fa42f9 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 20 Feb 2024 12:01:23 +1300 Subject: [PATCH 033/123] Fix file move in archiver tasks --- .../patches/ckanext-archiver/01_copy_function.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ckan/patches/ckanext-archiver/01_copy_function.patch diff --git a/ckan/patches/ckanext-archiver/01_copy_function.patch b/ckan/patches/ckanext-archiver/01_copy_function.patch new file mode 100644 index 00000000..727485a3 --- /dev/null +++ b/ckan/patches/ckanext-archiver/01_copy_function.patch @@ -0,0 +1,13 @@ +diff --git a/ckanext/archiver/tasks.py b/ckanext/archiver/tasks.py +index 06a1758..95fa968 100644 +--- a/ckanext/archiver/tasks.py ++++ b/ckanext/archiver/tasks.py +@@ -570,7 +570,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): + + # move the temp file to the resource's archival directory + saved_file = os.path.join(archive_dir, file_name) +- shutil.move(result['saved_file'], saved_file) ++ shutil.move(result['saved_file'], saved_file, copy_function=shutil.copy) + log.info('Going to do chmod: %s', saved_file) + try: + os.chmod(saved_file, 0o644) # allow other users to read it From 790f34625af2208554de74640ab805f7dcb7ebd8 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 20 Feb 2024 15:33:18 +1300 Subject: [PATCH 034/123] Update cron job to schedule embargo datasets to public visibility at 8am --- ckan/setup/dbca_ckan_cron_jobs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index cc3ea815..1b5653dd 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -3,7 +3,7 @@ 0 0 * * * /usr/sbin/logrotate -s /srv/app/logs/logrotate.status /etc/logrotate.d/dbca_ckan_rotate_logs # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run -# Midnight task to schedule embargo datasets to public visibility -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets +# 8am task to schedule embargo datasets to public visibility +0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets # Midnight report generation for archiver broken links 0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate \ No newline at end of file From 8fe85f781ff9d57540bf1d1149c79810887d1cdd Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 21 Feb 2024 06:22:28 +0800 Subject: [PATCH 035/123] Update devcontainer and launch configurations --- .devcontainer/devcontainer.json | 34 +++++++++++++++++++-------------- .vscode/launch.json | 6 +++--- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index edb45725..8e366f62 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,40 +21,46 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-python.vscode-pylance", + "ms-python.debugpy", "ms-python.autopep8", "ms-python.flake8", + "ms-python.vscode-pylance", "VisualStudioExptTeam.vscodeintellicode", "GitHub.vscode-pull-request-github", "eamodio.gitlens", "streetsidesoftware.code-spell-checker", + "streetsidesoftware.code-spell-checker-australian-english", "wholroyd.jinja", "GitHub.copilot", "GitHub.copilot-chat" ], // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.profiles.linux": { - "bash": { - "path": "/bin/bash" - } - }, + "terminal.integrated.defaultProfile.linux": "bash", + "python.defaultInterpreterPath": "python", "python.languageServer": "Pylance", "python.linting.enabled": true, - "python.linting.flake8Enabled": true, + "python.analysis.extraPaths": [ + "/srv/app/src" + ], + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + }, + "flake8.interpreter": [ + "python" + ], "flake8.args": [ "--max-line-length", - "120" + "160" + ], + "autopep8.interpreter": [ + "python" ], - "python.formatting.provider": "autopep8", "autopep8.args": [ "--max-line-length", - "120" + "160" ], - "cSpell.language": "en-GB", - "python.analysis.extraPaths": [ - "./app/src" - ] + "cSpell.language": "en-AU" } } }, diff --git a/.vscode/launch.json b/.vscode/launch.json index faac609b..68295596 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,7 @@ "configurations": [ { "name": "Python: CKAN Run & Debug", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pdb", "args": [ @@ -23,7 +23,7 @@ }, { "name": "Python: CKAN Jobs Worker Run & Debug", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pdb", "args": [ @@ -38,7 +38,7 @@ }, { "name": "Python: CKAN CLI Command Run & Debug", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pdb", "args": [ From a5f939819e0784f7f57c7e20304a91384376782d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 21 Feb 2024 10:03:06 +0800 Subject: [PATCH 036/123] 6710 Restrict adding of orgs and groups to sysadmin only --- ckan/config/dbca.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index cd0022f0..258cdf99 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -20,6 +20,12 @@ ckan.locale_default = en_AU ckan.display_timezone = Australia/Perth ckan.resource_formats = /srv/app/src/ckanext-dbca/ckanext/dbca/config/resource_formats.json +## Authorization Settings ###################################################### +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = false +ckan.auth.user_delete_organizations = false + ## CKAN Extensions configuration ############################################### # ckanext-spatial From be5ffe3a3a98f62d304f85ced58ce035a948d769 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 21 Feb 2024 11:48:11 +0800 Subject: [PATCH 037/123] Enabled activity plugin and activity_streams_email_notifications --- ckan/Dockerfile.worker | 3 +-- ckan/config/dbca.ini | 17 ++++++++++++++++- ckan/setup/dbca_ckan_cron_jobs | 4 +++- ckan/setup/dbca_start_ckan.sh.override | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 92a40a93..81430006 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -2,9 +2,8 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} # Override the default CKAN config plugins to only use the ones we need for the worker -ENV CKAN__PLUGINS="dbca datastore xloader scheming_datasets qa archiver report envvars" +ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" -ENV CKAN_SITE_URL=http://localhost:5000 # Install logrotate RUN apk add logrotate diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 258cdf99..7f2e8a41 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars +ckan.plugins = dbca activity image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view officedocs_view @@ -26,6 +26,10 @@ ckan.auth.user_create_organizations = false ckan.auth.user_delete_groups = false ckan.auth.user_delete_organizations = false +## Activity Streams Settings ################################################### +ckan.activity_streams_email_notifications = true + + ## CKAN Extensions configuration ############################################### # ckanext-spatial @@ -57,6 +61,17 @@ ckanext-archiver.cache_url_root=/resource_cache/ # ckanext-qa ckanext.qa.qsv_bin=/usr/local/bin/qsv +# ckanext-dbca +ckanext.dbca.spatial_data_path = /srv/app/spatial_data +ckanext.dbca.spatial_data_mappings = + { + "ibra.geojson": {"layer": "IBRA Regions", "code":"IWA_REG_CODE_7", "name":"IWA_REG_NAME_7"}, + "ibra-sub.geojson": {"layer": "IBRA Subregions)", "code":"IWA_SUB_CODE_7", "name":"IWA_SUB_NAME_7"}, + "imcra.geojson": {"layer": "IMCRA Regions", "code": "MESO_ABBR", "name":"MESO_NAME"}, + "lga-wa.geojson": {"layer": " Local Government Areas", "code":"LGA_TYPE", "name":"LGA_LGA_NAME"}, + "tenure.geojson": {"layer": "DBCA Managed Tenure", "code":"LEG_TENURE", "name":"LEG_NAME"} + } + ## Logging configuration [loggers] diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 1b5653dd..e888624e 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -6,4 +6,6 @@ # 8am task to schedule embargo datasets to public visibility 0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets # Midnight report generation for archiver broken links -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate \ No newline at end of file +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate +# 8am task to send CKAN email notifications +0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index f4b19028..d9e3bbff 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -32,7 +32,7 @@ then done fi -# Revert back touse DBCA config file +# Revert back to use DBCA config file export CKAN_INI=$APP_DIR/config/dbca.ini if [ $? -eq 0 ] From 46812af5c2560b1ca6a18595a1fa8ca35a9b5239 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 22 Feb 2024 11:54:40 +0800 Subject: [PATCH 038/123] Fix file move and remove in archiver tasks --- .../01_copy_archive_resource.patch | 27 +++++++++++++++++++ .../ckanext-archiver/01_copy_function.patch | 13 --------- 2 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 ckan/patches/ckanext-archiver/01_copy_archive_resource.patch delete mode 100644 ckan/patches/ckanext-archiver/01_copy_function.patch diff --git a/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch new file mode 100644 index 00000000..4a984354 --- /dev/null +++ b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch @@ -0,0 +1,27 @@ +index 06a1758..08a3ba6 100644 +--- a/ckanext/archiver/tasks.py ++++ b/ckanext/archiver/tasks.py +@@ -568,14 +568,14 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): + except Exception: + file_name = "resource" + +- # move the temp file to the resource's archival directory +- saved_file = os.path.join(archive_dir, file_name) +- shutil.move(result['saved_file'], saved_file) +- log.info('Going to do chmod: %s', saved_file) + try: +- os.chmod(saved_file, 0o644) # allow other users to read it ++ # move the temp file to the resource's archival directory ++ saved_file = os.path.join(archive_dir, file_name) ++ shutil.copy(result['saved_file'], saved_file) ++ log.info('File moved : %s', saved_file) ++ os.remove(result['saved_file']) + except Exception as e: +- log.error('chmod failed %s: %s', saved_file, e) ++ log.error("move the temp file to the resource's archival directory failed %s: %s", saved_file, e) + raise + log.info('Archived resource as: %s', saved_file) + + +~ +~ \ No newline at end of file diff --git a/ckan/patches/ckanext-archiver/01_copy_function.patch b/ckan/patches/ckanext-archiver/01_copy_function.patch deleted file mode 100644 index 727485a3..00000000 --- a/ckan/patches/ckanext-archiver/01_copy_function.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/ckanext/archiver/tasks.py b/ckanext/archiver/tasks.py -index 06a1758..95fa968 100644 ---- a/ckanext/archiver/tasks.py -+++ b/ckanext/archiver/tasks.py -@@ -570,7 +570,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): - - # move the temp file to the resource's archival directory - saved_file = os.path.join(archive_dir, file_name) -- shutil.move(result['saved_file'], saved_file) -+ shutil.move(result['saved_file'], saved_file, copy_function=shutil.copy) - log.info('Going to do chmod: %s', saved_file) - try: - os.chmod(saved_file, 0o644) # allow other users to read it From 4bc07e7deaf7a68f3959230a465c37c4488bb71b Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 23 Feb 2024 10:20:28 +1300 Subject: [PATCH 039/123] Fix file copying in ckanext-archiver --- .../ckanext-archiver/01_copy_archive_resource.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch index 4a984354..ad48a5c6 100644 --- a/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch +++ b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch @@ -1,7 +1,7 @@ -index 06a1758..08a3ba6 100644 +index 06a1758..eac41f3 100644 --- a/ckanext/archiver/tasks.py +++ b/ckanext/archiver/tasks.py -@@ -568,14 +568,14 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): +@@ -568,14 +568,13 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): except Exception: file_name = "resource" @@ -13,15 +13,15 @@ index 06a1758..08a3ba6 100644 - os.chmod(saved_file, 0o644) # allow other users to read it + # move the temp file to the resource's archival directory + saved_file = os.path.join(archive_dir, file_name) -+ shutil.copy(result['saved_file'], saved_file) -+ log.info('File moved : %s', saved_file) ++ shutil.copyfile(result['saved_file'], saved_file) + os.remove(result['saved_file']) except Exception as e: - log.error('chmod failed %s: %s', saved_file, e) -+ log.error("move the temp file to the resource's archival directory failed %s: %s", saved_file, e) ++ log.error("moving the temp file to the resource's archival directory failed %s: %s", saved_file, e) raise log.info('Archived resource as: %s', saved_file) +~ ~ ~ \ No newline at end of file From 4a5c62692c2a0afd2f23cd01c7e5db6ebff500e6 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 23 Feb 2024 09:13:20 +0800 Subject: [PATCH 040/123] Created patch for CKAN core bug https://github.com/ckan/ckan/pull/8020 --- ckan/patches/00_test_test.patch | 1 - ...ve_custom_org_type_in_user_activity_streams.patch | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) delete mode 100644 ckan/patches/00_test_test.patch create mode 100644 ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch diff --git a/ckan/patches/00_test_test.patch b/ckan/patches/00_test_test.patch deleted file mode 100644 index 30d74d25..00000000 --- a/ckan/patches/00_test_test.patch +++ /dev/null @@ -1 +0,0 @@ -test \ No newline at end of file diff --git a/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch b/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch new file mode 100644 index 00000000..a05ced5c --- /dev/null +++ b/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch @@ -0,0 +1,12 @@ +diff --git a/ckanext/activity/templates/snippets/stream.html b/ckanext/activity/templates/snippets/stream.html +index 52c0d4b49..270a5f22f 100644 +--- a/ckanext/activity/templates/snippets/stream.html ++++ b/ckanext/activity/templates/snippets/stream.html +@@ -14,6 +14,7 @@ + {% endmacro %} + + {% macro organization(activity) %} ++ {% set group_type = group_type or (activity.data.group.type if (activity.data.group and activity.data.group.type) else 'organization') %} + + {{ activity.data.group.title if activity.data.group else _('unknown') }} + From fc203b7139f327f075efef8707125d3ef596afb0 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 23 Feb 2024 16:21:30 +1300 Subject: [PATCH 041/123] Removed logrotate and consolidated logs into one ckan.log --- ckan/Dockerfile.worker | 5 ----- ckan/docker-entrypoint.d/02_setup_dbca.sh | 2 -- ckan/setup/dbca_ckan_cron_jobs | 2 -- ckan/setup/dbca_ckan_rotate_logs | 10 ---------- ckan/supervisor/ckan_cron_jobs.conf | 2 +- ckan/supervisor/ckan_worker_bulk.conf | 2 +- ckan/supervisor/ckan_worker_default.conf | 2 +- ckan/supervisor/ckan_worker_priority.conf | 2 +- 8 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 ckan/setup/dbca_ckan_rotate_logs diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 81430006..18b681fe 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -5,11 +5,6 @@ FROM ${CKAN_IMAGE} ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" -# Install logrotate -RUN apk add logrotate -# Create logrotate config file for app logs -COPY setup/dbca_ckan_rotate_logs /etc/logrotate.d/dbca_ckan_rotate_logs - ## Supervisor config COPY supervisor/*.conf /etc/supervisord.d diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index c2564cb3..7b2196dd 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -3,8 +3,6 @@ ## Create logs folder/files and set permissions mkdir -p $APP_DIR/logs touch $APP_DIR/logs/ckan.log -touch $APP_DIR/logs/ckan-worker.log -touch $APP_DIR/logs/ckan-crons-jobs.log chown -R ckan:ckan $APP_DIR/logs ## Create archive folder and set permissions diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index e888624e..c25ee32b 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,6 +1,4 @@ # Crontab for CKAN cron jobs -# Midnight task to rotate CKAN app logs every day -0 0 * * * /usr/sbin/logrotate -s /srv/app/logs/logrotate.status /etc/logrotate.d/dbca_ckan_rotate_logs # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # 8am task to schedule embargo datasets to public visibility diff --git a/ckan/setup/dbca_ckan_rotate_logs b/ckan/setup/dbca_ckan_rotate_logs deleted file mode 100644 index f1af6a96..00000000 --- a/ckan/setup/dbca_ckan_rotate_logs +++ /dev/null @@ -1,10 +0,0 @@ -/srv/app/logs/ckan-worker.log /srv/app/logs/ckan-crons-jobs.log { - su ckan ckan - daily - rotate 7 - missingok - notifempty - compress - delaycompress - copytruncate -} \ No newline at end of file diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index ed5e3fa9..ca3de39a 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan-crons-jobs.log +stderr_logfile=/srv/app/logs/ckan.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf index b79065c1..cf513309 100644 --- a/ckan/supervisor/ckan_worker_bulk.conf +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan-worker.log +stderr_logfile=/srv/app/logs/ckan.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index 354bfc1b..3e2ccd19 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan-worker.log +stderr_logfile=/srv/app/logs/ckan.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf index 195ad982..0c73f6ab 100644 --- a/ckan/supervisor/ckan_worker_priority.conf +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan-worker.log +stderr_logfile=/srv/app/logs/ckan.log ; Make sure that the worker is started on system start and automatically From d101e6587adad7f9c070ee8f4ddbdade94a37443 Mon Sep 17 00:00:00 2001 From: MarkCalvert <37602611+MarkCalvert@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:03:08 +1300 Subject: [PATCH 042/123] [BIO-6525] Add DBCA spatial autocomplete feature (#10) * [BIO-6525] Add spatial_data directory and load spatial data for dbca plugin * Update devcontainer and .gitignore files, and add spatial_data directory with README.md --- .devcontainer/devcontainer.json | 3 +-- .gitignore | 6 ++++- ckan/docker-entrypoint.d/02_setup_dbca.sh | 5 ++++ docker-compose.dev.yml | 1 + spatial_data/README.MD | 28 +++++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 spatial_data/README.MD diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8e366f62..3a780382 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -36,12 +36,11 @@ ], // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.defaultProfile.linux": "bash", "python.defaultInterpreterPath": "python", "python.languageServer": "Pylance", "python.linting.enabled": true, "python.analysis.extraPaths": [ - "/srv/app/src" + "./app/src/ckan" ], "[python]": { "editor.defaultFormatter": "ms-python.autopep8" diff --git a/.gitignore b/.gitignore index b21eec73..7f3e9f9e 100755 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,8 @@ _solr/schema.xml _src/* local/* .env -dbca/ \ No newline at end of file + +# DBCA +src/ +dbca/ +spatial_data/*.geojson \ No newline at end of file diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 7b2196dd..76166c99 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -39,6 +39,11 @@ if [[ $CKAN__PLUGINS == *"doi"* ]]; then ckan -c $CKAN_INI doi initdb fi +if [[ $CKAN__PLUGINS == *"dbca"* ]]; then + ckan -c $CKAN_INI db upgrade -p dbca + ckan -c $CKAN_INI dbca load_spatial_data +fi + # if [[ $CKAN__PLUGINS == *"harvest"* ]]; then # ckan -c $CKAN_INI db upgrade -p harvest # fi \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 86be507b..559e0a47 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -34,6 +34,7 @@ services: - ./src:/srv/app/src_extensions - ./ckan/config:/srv/app/config - ckan_logs:/srv/app/logs + - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/spatial_data/README.MD b/spatial_data/README.MD new file mode 100644 index 00000000..ed982eb8 --- /dev/null +++ b/spatial_data/README.MD @@ -0,0 +1,28 @@ +# Spatial Data + +This directory is used to store GeoJSON files that will be loaded from docker entrypoint file `02_setup_dbca.sh` using the CLI command `ckan -c $CKAN_INI dbca load_spatial_data` if the mapping exists in `ckanext.dbca.spatial_data_mapping`. + +## Usage + +To add a new GeoJSON file, simply place it in this directory. The container will automatically load the file if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. + +Please ensure that the GeoJSON file follows the correct format and contains valid spatial data. + +## Mapping + +The mapping between the GeoJSON files and the data fields in the CKAN database is defined in the `ckanext.dbca.spatial_data_mapping` configuration. Make sure to update the mapping accordingly if you add or modify any GeoJSON files. + +Each mapping consists of a filename, layer field, a code field, and a name field. + +The mappings are defined as a dictionary where the key is the filename of the GeoJSON file and the value is another dictionary containing the layer, code, and name fields. + +Example: +``` +{ + "ibra.geojson": {"layer": "IBRA Regions", "code":"IWA_REG_CODE_7", "name":"IWA_REG_NAME_7"}, + "ibra-sub.geojson": {"layer": "IBRA Subregions)", "code":"IWA_SUB_CODE_7", "name":"IWA_SUB_NAME_7"}, + "imcra.geojson": {"layer": "IMCRA Regions", "code": "MESO_ABBR", "name":"MESO_NAME"}, + "lga-wa.geojson": {"layer": " Local Government Areas", "code":"LGA_TYPE", "name":"LGA_LGA_NAME"}, + "tenure.geojson": {"layer": "DBCA Managed Tenure", "code":"LEG_TENURE", "name":"LEG_NAME"} +} +``` From 967ca34697fc167d458dee483d2d12d5ebd601dd Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 28 Feb 2024 08:45:35 +0800 Subject: [PATCH 043/123] Update CKAN config log file handler to use the worker log file and the supervisor processes --- ckan/Dockerfile.worker | 2 ++ ckan/config/dbca.ini | 2 +- ckan/docker-entrypoint.d/02_setup_dbca.sh | 1 + ckan/supervisor/ckan_cron_jobs.conf | 2 +- ckan/supervisor/ckan_worker_bulk.conf | 2 +- ckan/supervisor/ckan_worker_default.conf | 2 +- ckan/supervisor/ckan_worker_priority.conf | 2 +- 7 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 18b681fe..676ae428 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -4,6 +4,8 @@ FROM ${CKAN_IMAGE} # Override the default CKAN config plugins to only use the ones we need for the worker ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" +# Update the CKAN config log file handler to use the worker log file +RUN ckan config-tool $CKAN_INI -s handler_fileHandler "args = ('/srv/app/logs/ckan-worker.log', 'midnight', 1, 7)" ## Supervisor config COPY supervisor/*.conf /etc/supervisord.d diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 7f2e8a41..13119baa 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -113,7 +113,7 @@ formatter = generic [handler_fileHandler] class = handlers.TimedRotatingFileHandler -args = ('/srv/app/logs/ckan.log', 'midnight', 1, 0) +args = ('/srv/app/logs/ckan.log', 'midnight', 1, 7) level = NOTSET formatter = generic diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 76166c99..7301369b 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -3,6 +3,7 @@ ## Create logs folder/files and set permissions mkdir -p $APP_DIR/logs touch $APP_DIR/logs/ckan.log +touch $APP_DIR/logs/ckan-worker.log chown -R ckan:ckan $APP_DIR/logs ## Create archive folder and set permissions diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index ca3de39a..bc4cbf3b 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf index cf513309..b79065c1 100644 --- a/ckan/supervisor/ckan_worker_bulk.conf +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index 3e2ccd19..354bfc1b 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf index 0c73f6ab..195ad982 100644 --- a/ckan/supervisor/ckan_worker_priority.conf +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -24,7 +24,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stderr_logfile=/srv/app/logs/ckan.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically From f9deecff01f7b8f054112c8803d9411bdcf40e6e Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 5 Mar 2024 11:38:31 +1300 Subject: [PATCH 044/123] Added forked version of ckanext-showcase --- ckan/setup/dbca_requirements.sh | 12 ++++++++---- src/dbca_install_extensions.sh | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index b5847a69..f8a5c86e 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -28,9 +28,9 @@ pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanex pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext-report pip3 install -r ${SRC_DIR}/ckanext-report/requirements.txt -# Showcase -pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.6.1#egg=ckanext-showcase -pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt +# # Showcase +# pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.6.1#egg=ckanext-showcase +# pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt # Scheming pip3 install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming @@ -74,4 +74,8 @@ pip3 install -e git+https://github.com/dbca-wa/ckanext-qa.git@develop#egg=ckanex pip3 install -r ${SRC_DIR}/ckanext-qa/requirements.txt # Office Docs -pip3 install -e git+https://github.com/dbca-wa/ckanext-officedocs.git@develop#egg=ckanext-officedocs \ No newline at end of file +pip3 install -e git+https://github.com/dbca-wa/ckanext-officedocs.git@develop#egg=ckanext-officedocs + +# Showcase +pip3 install -e git+https://github.com/dbca-wa/ckanext-showcase.git@develop#egg=ckanext-showcase +pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt \ No newline at end of file diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index 1d21f24e..3e5473de 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -13,5 +13,7 @@ git clone https://github.com/dbca-wa/ckanext-qa.git sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt # Office Docs git clone https://github.com/dbca-wa/ckanext-officedocs.git +# Showcase +git clone https://github.com/dbca-wa/ckanext-showcase.git echo "Ready to build project: ahoy build" From 0c99056aa2ed48f52a0a35aaf7a9a411bf4b366e Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 6 Mar 2024 11:47:04 +0800 Subject: [PATCH 045/123] Removed downloading ckan-crons-jobs.log --- .ahoy.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index 23f2621c..29d7b8cc 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -164,11 +164,6 @@ commands: echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" - - LOG_FILE="ckan-crons-jobs.log" - echo "Downloading $LOG_FILE" - FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" - curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" LOG_FILE="ckan-worker.log" echo "Downloading $LOG_FILE" From 62668b17887dce055f5704d3e97f26d25253743c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 12 Mar 2024 08:54:03 +0800 Subject: [PATCH 046/123] Removed bracket --- ckan/config/dbca.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 13119baa..20b33a67 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -66,7 +66,7 @@ ckanext.dbca.spatial_data_path = /srv/app/spatial_data ckanext.dbca.spatial_data_mappings = { "ibra.geojson": {"layer": "IBRA Regions", "code":"IWA_REG_CODE_7", "name":"IWA_REG_NAME_7"}, - "ibra-sub.geojson": {"layer": "IBRA Subregions)", "code":"IWA_SUB_CODE_7", "name":"IWA_SUB_NAME_7"}, + "ibra-sub.geojson": {"layer": "IBRA Subregions", "code":"IWA_SUB_CODE_7", "name":"IWA_SUB_NAME_7"}, "imcra.geojson": {"layer": "IMCRA Regions", "code": "MESO_ABBR", "name":"MESO_NAME"}, "lga-wa.geojson": {"layer": " Local Government Areas", "code":"LGA_TYPE", "name":"LGA_LGA_NAME"}, "tenure.geojson": {"layer": "DBCA Managed Tenure", "code":"LEG_TENURE", "name":"LEG_NAME"} From b5da04c0717b454055ede3af35443547cc8fc59c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 13 Mar 2024 10:49:19 +0800 Subject: [PATCH 047/123] Add DOI plugin to CKAN worker Dockerfile --- ckan/Dockerfile.worker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 676ae428..2ee7d549 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -2,7 +2,7 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} # Override the default CKAN config plugins to only use the ones we need for the worker -ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report envvars" +ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" # Update the CKAN config log file handler to use the worker log file RUN ckan config-tool $CKAN_INI -s handler_fileHandler "args = ('/srv/app/logs/ckan-worker.log', 'midnight', 1, 7)" From 80cacc51f58b0aa13bd358eac10b2cdb6161951c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 13 Mar 2024 10:49:44 +0800 Subject: [PATCH 048/123] Update dependencies for DBCA project to use DOI forked repo --- ckan/setup/dbca_requirements.sh | 7 +++++-- src/dbca_install_extensions.sh | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index f8a5c86e..5486afb6 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -56,7 +56,7 @@ pip3 install -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckane ## 3rd Party ## # DOI -pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi +# pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi ## DBCA Project ## @@ -64,6 +64,9 @@ pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10# # DBCA pip3 install -e git+https://github.com/dbca-wa/ckanext-dbca.git@develop#egg=ckanext-dbca +# DOI +pip3 install -e git+https://github.com/dbca-wa/ckanext-doi@develop#egg=ckanext-doi + # QA # Install qsv dependency for extension ckanext-qa wget -O /tmp/qsv.zip https://github.com/jqnatividad/qsv/releases/download/0.110.0/qsv-0.110.0-x86_64-unknown-linux-musl.zip @@ -78,4 +81,4 @@ pip3 install -e git+https://github.com/dbca-wa/ckanext-officedocs.git@develop#eg # Showcase pip3 install -e git+https://github.com/dbca-wa/ckanext-showcase.git@develop#egg=ckanext-showcase -pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt \ No newline at end of file +pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index 3e5473de..a279bcf2 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -15,5 +15,7 @@ sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver git clone https://github.com/dbca-wa/ckanext-officedocs.git # Showcase git clone https://github.com/dbca-wa/ckanext-showcase.git +# DOI +git clone https://github.com/dbca-wa/ckanext-doi echo "Ready to build project: ahoy build" From cc095f1323a6dc81d49ae76c5c24bc1d9a1c0172 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 13 Mar 2024 12:40:01 +0100 Subject: [PATCH 049/123] Bump 2.10 version to 2.10.4 --- ckan/Dockerfile | 2 +- ckan/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 5a1ee544..f9dae59a 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -FROM ckan/ckan-base:2.10.3 +FROM ckan/ckan-base:2.10.4 # Install any extensions needed by your CKAN instance # See Dockerfile.dev for more details and examples diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 52ed9753..1f70363b 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM ckan/ckan-dev:2.10.3 +FROM ckan/ckan-dev:2.10.4 # Install any extensions needed by your CKAN instance # - Make sure to add the plugins to CKAN__PLUGINS in the .env file From 5981a97a754bbedf4abfe5fe032cc3c2beeb5c90 Mon Sep 17 00:00:00 2001 From: Oleg Butuzov Date: Mon, 18 Mar 2024 17:18:19 +0200 Subject: [PATCH 050/123] docs: mention of ckan.ini in `docker compose` command --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f5548a7c..5f689cc9 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ See [CKAN Images](#ckan-images) for more details of what happens when using deve You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tutorial.html#creating-a-new-extension) instructions to create a CKAN extension, only executing the command inside the CKAN container and setting the mounted `src/` folder as output: - docker compose -f docker-compose.dev.yml exec ckan-dev /bin/sh -c "ckan generate extension --output-dir /srv/app/src_extensions" - + docker compose -f docker-compose.dev.yml exec ckan-dev /bin/sh -c "ckan -c /srv/app/ckan.ini generate extension --output-dir /srv/app/src_extensions" + ![Screenshot 2023-02-22 at 1 45 55 pm](https://user-images.githubusercontent.com/54408245/220623568-b4e074c7-6d07-4d27-ae29-35ce70961463.png) From 6c61c6c71be4c2ba2e0431cb7f58edccaf1f119a Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 19 Mar 2024 17:26:01 +1300 Subject: [PATCH 051/123] Add Solr reindexing command to .ahoy.yml --- .ahoy.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.ahoy.yml b/.ahoy.yml index 29d7b8cc..68473805 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -125,9 +125,14 @@ commands: cmd: | if [ -e "$@" ] ; then docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@" + ahoy reindex-solr else echo "Provided sql file" "$@" "does not exist" fi + reindex-solr: + usage: Re-runs the Solr index, e.g. after running `db-import` command + cmd: ahoy run "ckan search-index rebuild" + db-dump: usage: Dump data out into a file. `ahoy db-dump local.dump` cmd: docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_dump -U $CKAN_DB_USER -d $CKAN_DB --format=custom -v' > "$@" From acc18136c65da7e76a54d5da94c945c0a1a16015 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 20 Mar 2024 10:09:05 +1300 Subject: [PATCH 052/123] Removed patch because the latest CKAN release 2.10.4 includes the fix --- ...ve_custom_org_type_in_user_activity_streams.patch | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch diff --git a/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch b/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch deleted file mode 100644 index a05ced5c..00000000 --- a/ckan/patches/ckan/8020_preserve_custom_org_type_in_user_activity_streams.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/ckanext/activity/templates/snippets/stream.html b/ckanext/activity/templates/snippets/stream.html -index 52c0d4b49..270a5f22f 100644 ---- a/ckanext/activity/templates/snippets/stream.html -+++ b/ckanext/activity/templates/snippets/stream.html -@@ -14,6 +14,7 @@ - {% endmacro %} - - {% macro organization(activity) %} -+ {% set group_type = group_type or (activity.data.group.type if (activity.data.group and activity.data.group.type) else 'organization') %} - - {{ activity.data.group.title if activity.data.group else _('unknown') }} - From a24fc162910f57f8808375de3a7782d518542449 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 20 Mar 2024 10:42:07 +1300 Subject: [PATCH 053/123] Add import-db command to .ahoy.yml and define environment variables for Azure Postgres --- .ahoy.yml | 28 ++++++++++++++++++++++++++++ .env.dbca | 10 ++++++++++ scripts/messages.sh | 18 ++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 scripts/messages.sh diff --git a/.ahoy.yml b/.ahoy.yml index 68473805..66cf740b 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -129,6 +129,34 @@ commands: else echo "Provided sql file" "$@" "does not exist" fi + import-db: + usage: Import the DB from an environment into your local dev environment | ahoy import-db ENVIRONMENT + cmd: | + source ./scripts/messages.sh + # + # Check the correct arguments are supplied + # + if [[ "$1" != "staging" ]] && [[ "$1" != "production" ]]; then + error "Invalid environment. Please use 'staging' or 'production'. eg ahoy import-db ENVIRONMENT" + exit 1 + fi + + ENVIRONMENT=$1 + info "Dumping DB from $ENVIRONMENT" + if [[ "$1" == "staging" ]]; then + docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'cd /tmp && pg_dump "postgresql://'"$AZURE_POSTGRES_STAGING_USER"':'"$AZURE_POSTGRES_STAGING_PASSWORD"'@'"$AZURE_POSTGRES_STAGING_HOST"'/'"$AZURE_POSTGRES_STAGING_DB"'" --format=custom -v > latest.dump' + else + docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'cd /tmp && pg_dump "postgresql://'"$AZURE_POSTGRES_PROD_USER"':'"$AZURE_POSTGRES_PROD_PASSWORD"'@'"$AZURE_POSTGRES_PROD_HOST"'/'"$AZURE_POSTGRES_PROD_DB"'" --exclude-table=dbca_spatial --format=custom -v > latest.dump' + fi + + info "Importing DB from $ENVIRONMENT" + docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists --no-owner --no-privileges -v < /tmp/latest.dump' + info "Rebuilding Solr Index" + ahoy reindex-solr + + success "Database import successful" + + reindex-solr: usage: Re-runs the Solr index, e.g. after running `db-import` command cmd: ahoy run "ckan search-index rebuild" diff --git a/.env.dbca b/.env.dbca index f39efa2f..c8e7bb3e 100644 --- a/.env.dbca +++ b/.env.dbca @@ -93,3 +93,13 @@ DOCKER_COMPOSE=docker-compose.dev.yml # Azure File Share Storage AZURE_FILE_SHARE_URL= AZURE_FILE_SHARE_SAS_TOKEN= + +# Azure Postgres +AZURE_POSTGRES_STAGING_USER= +AZURE_POSTGRES_STAGING_PASSWORD= +AZURE_POSTGRES_STAGING_HOST= +AZURE_POSTGRES_STAGING_DB= +AZURE_POSTGRES_PROD_USER= +AZURE_POSTGRES_PROD_PASSWORD= +AZURE_POSTGRES_PROD_HOST= +AZURE_POSTGRES_PROD_DB= \ No newline at end of file diff --git a/scripts/messages.sh b/scripts/messages.sh new file mode 100644 index 00000000..c42a981a --- /dev/null +++ b/scripts/messages.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' + +success() { + echo -e "${GREEN} [√] ${1}${NC}" +} + +info() { + echo -e "${YELLOW} [-] ${1}${NC}" +} + +error() { + echo -e "\n${RED} [x] ${1}${NC}" +} From 8416fdc0274ce3686d0ec1d4979113e9aa80e0f4 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 15:49:04 +1300 Subject: [PATCH 054/123] Update NGINX configuration and add CKAN server name --- .env.dbca | 1 + nginx/Dockerfile | 3 ++- nginx/setup/default.conf.template | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.env.dbca b/.env.dbca index c8e7bb3e..b17148bb 100644 --- a/.env.dbca +++ b/.env.dbca @@ -74,6 +74,7 @@ CKANEXT__XLOADER__JOBS_DB__URI=$CKAN_SQLALCHEMY_URL # NGINX NGINX_PORT=80 NGINX_SSLPORT=443 +CKAN_SERVER_NAME=localhost # Extensions diff --git a/nginx/Dockerfile b/nginx/Dockerfile index aed247e3..47e072d3 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -9,7 +9,7 @@ RUN apk update --no-cache && \ COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html COPY setup/default.conf ${NGINX_DIR}/conf.d/ -COPY setup/default.conf.template ${NGINX_DIR}/templates/ +COPY setup/default.conf.template ${NGINX_DIR}/ RUN mkdir -p ${NGINX_DIR}/certs @@ -29,4 +29,5 @@ ENTRYPOINT \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ /usr/sbin/crond && \ + mkdir -p ${NGINX_DIR}/templates && mv ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ /docker-entrypoint.sh nginx -g 'daemon off;' diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index ca80d5bf..c5cdafcf 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -3,7 +3,7 @@ server { #listen [::]:80; listen 443 ssl; listen [::]:443 ssl; - server_name localhost; + server_name ${CKAN_SERVER_NAME}; ssl_certificate /etc/nginx/certs/ckan-local.crt; ssl_certificate_key /etc/nginx/certs/ckan-local.key; From a828a25e650a96d7536ab3a136dc755ea1b65815 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 15:51:20 +1300 Subject: [PATCH 055/123] Update nginx Dockerfile to copy default.conf.template to templates directory --- nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 47e072d3..4b661d79 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -29,5 +29,5 @@ ENTRYPOINT \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ /usr/sbin/crond && \ - mkdir -p ${NGINX_DIR}/templates && mv ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ + mkdir -p ${NGINX_DIR}/templates && cp ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ /docker-entrypoint.sh nginx -g 'daemon off;' From f4566ac116b32da518e9bb7499653aaf09b9c18d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:06:03 +1300 Subject: [PATCH 056/123] Commented out the creation of the templates directory and the copy command in the Dockerfile --- nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 4b661d79..ddd80fca 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -29,5 +29,5 @@ ENTRYPOINT \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ /usr/sbin/crond && \ - mkdir -p ${NGINX_DIR}/templates && cp ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ + # mkdir -p ${NGINX_DIR}/templates && cp ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ /docker-entrypoint.sh nginx -g 'daemon off;' From 60dfe8bd51fe3a743c083f0f76def98aa47510b4 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:06:53 +1300 Subject: [PATCH 057/123] Update Dockerfile to move default.conf.template to templates directory --- nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index ddd80fca..c319e099 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -9,7 +9,7 @@ RUN apk update --no-cache && \ COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html COPY setup/default.conf ${NGINX_DIR}/conf.d/ -COPY setup/default.conf.template ${NGINX_DIR}/ +COPY setup/default.conf.template ${NGINX_DIR}/templates RUN mkdir -p ${NGINX_DIR}/certs From ce4d4b55acc0002ccdcd6b772b0d8209f2edbfe7 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:15:44 +1300 Subject: [PATCH 058/123] Remove unnecessary code in nginx Dockerfile --- nginx/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index c319e099..2f2f94e9 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -29,5 +29,4 @@ ENTRYPOINT \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ /usr/sbin/crond && \ - # mkdir -p ${NGINX_DIR}/templates && cp ${NGINX_DIR}/default.conf.template ${NGINX_DIR}/templates/default.conf.template && \ /docker-entrypoint.sh nginx -g 'daemon off;' From fdd5a24efcbb1c4391708da03a2c5b12d75ea7b3 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:24:02 +1300 Subject: [PATCH 059/123] Update Dockerfile to fix file path in COPY command --- nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 2f2f94e9..aed247e3 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -9,7 +9,7 @@ RUN apk update --no-cache && \ COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf COPY setup/index.html /usr/share/nginx/html/index.html COPY setup/default.conf ${NGINX_DIR}/conf.d/ -COPY setup/default.conf.template ${NGINX_DIR}/templates +COPY setup/default.conf.template ${NGINX_DIR}/templates/ RUN mkdir -p ${NGINX_DIR}/certs From f8ecaf2737bcab3e970ba81719d050b9037c4e5e Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:55:23 +1300 Subject: [PATCH 060/123] Update nginx setup to listen on port 80 and 443 --- nginx/setup/default.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index 61359b65..5b03f96d 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -1,6 +1,6 @@ server { - #listen 80; - #listen [::]:80; + listen 80; + listen [::]:80; listen 443 ssl; listen [::]:443 ssl; server_name localhost; From 0164cb691e29b9eba2a6af19deed7e519b478699 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 28 Mar 2024 16:59:13 +1300 Subject: [PATCH 061/123] Enable HTTP and HTTPS listening on port 80 in nginx configuration --- nginx/setup/default.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index c5cdafcf..76e8aab7 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -1,6 +1,6 @@ server { - #listen 80; - #listen [::]:80; + listen 80; + listen [::]:80; listen 443 ssl; listen [::]:443 ssl; server_name ${CKAN_SERVER_NAME}; From b37c160072a0f0075ca731202cd1244fe6170b3c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 3 Apr 2024 16:49:41 +1300 Subject: [PATCH 062/123] [BIO-WA 6358] Enable and configure SSO extension ckanext-saml2auth --- ckan/Dockerfile | 4 + ckan/Dockerfile.dev | 4 + ckan/config/dbca.ini | 34 +++++- ckan/saml/dbca_idp.xml | 180 ++++++++++++++++++++++++++++++++ ckan/saml/dbca_staging_sp.xml | 7 ++ ckan/setup/dbca_requirements.sh | 2 + docker-compose.dev.yml | 2 +- docker-compose.yml | 1 + 8 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 ckan/saml/dbca_idp.xml create mode 100644 ckan/saml/dbca_staging_sp.xml diff --git a/ckan/Dockerfile b/ckan/Dockerfile index aa15d632..230a46d2 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -24,6 +24,10 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +# ckanext-saml2auth dependencies +RUN apk add xmlsec +COPY saml ${APP_DIR}/saml + # Override the default start_ckan.sh script COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh RUN chmod +x ${APP_DIR}/start_ckan.sh diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 33a1b2d5..7f958ff2 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,6 +57,10 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +# ckanext-saml2auth dependencies +RUN apk add xmlsec +COPY saml ${APP_DIR}/saml + # Override the default dbca_start_ckan_development.sh script COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh RUN chmod +x ${APP_DIR}/start_ckan_development.sh diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 20b33a67..10153a66 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,7 +10,7 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca activity image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars +ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars ## Resource Views Settings ##################################################### ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view officedocs_view @@ -72,6 +72,38 @@ ckanext.dbca.spatial_data_mappings = "tenure.geojson": {"layer": "DBCA Managed Tenure", "code":"LEG_TENURE", "name":"LEG_NAME"} } +# ckanext-saml2auth +# Specifies the metadata location type +# Options: local or remote +ckanext.saml2auth.idp_metadata.location = local + +# Path to a local file accessible on the server the service runs on +# Ignore this config if the idp metadata location is set to: remote +ckanext.saml2auth.idp_metadata.local_path = /srv/app/saml/dbca_idp.xmlå + +# Corresponding SAML user field for firstname +ckanext.saml2auth.user_firstname = givenname + +# Corresponding SAML user field for lastname +ckanext.saml2auth.user_lastname = surname + +# Corresponding SAML user field for fullname +# (Optional: Can be used as an alternative to firstname + lastname) +#ckanext.saml2auth.user_fullname = fullname + +# Corresponding SAML user field for email +ckanext.saml2auth.user_email = emailaddress + +# Optional: Corresponding SAML user field for groups + +# Configuration setting that enables CKAN's internal register/login functionality as well +# Default: False +ckanext.saml2auth.enable_ckan_internal_login = True + +# Entity ID (also know as Issuer) +# Define the entity ID. Default is urn:mace:umu.se:saml:ckan:sp +ckanext.saml2auth.entity_id = urn:mace:umu.se:saml:ckan_dbca_staging:sp + ## Logging configuration [loggers] diff --git a/ckan/saml/dbca_idp.xml b/ckan/saml/dbca_idp.xml new file mode 100644 index 00000000..63102981 --- /dev/null +++ b/ckan/saml/dbca_idp.xml @@ -0,0 +1,180 @@ + + + + + + + + + + + + + jO1rbEKIIubNrKyefNwA/DLzTF9gtJ5oO2jTfaYB/tw= + + + ntzlyGOD0fP/IIw/R2IvJb/UMx7dBlwByXNiL9TEH318Jr5iF9vsGTXef5WKUQLy7ZGm9FkKJu9221kWXjpl2az+cfeRduuvurjYIsIHu2IVoz04wi7e6EUjzHwx9oJxnpZ4orfiGvgtkrLH0ai8qkNzSsztqHQMsV1tAYQeYtq0BC890nxl9KbLEO3IPhEozaQWZRsWMYQS8kFehPkJHDzcg6ZYXB+CglO4xga6iYacJScfrP4Mf7zc7oYmOxdPPUalBAXDam5bAQVpJrs3xe0p+4yPg3buD36Uy3alhSaXui6AoNRIFATb2JN0k1BFGQBLROVhflsfPKAit/rmJg== + + + MIIC8DCCAdigAwIBAgIQH0zLxpk8ZIpL6FjEf3kvkTANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylNaWNyb3NvZnQgQXp1cmUgRmVkZXJhdGVkIFNTTyBDZXJ0aWZpY2F0ZTAeFw0yNDAzMjgwNzQ2NTlaFw0yNzAzMjgwNzQ2NDhaMDQxMjAwBgNVBAMTKU1pY3Jvc29mdCBBenVyZSBGZWRlcmF0ZWQgU1NPIENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArXs7OiS8m1ZM4y5yQv4Cze8uDkl44bI1zx3Pl+dQRcPHo7KmY8RKAzFgGCsxDyFqgzIPX9vTotBoqWFmj+U2FimzsFPV9GJVfJkmRbxG5NqKir8jlrziZwEe/3+8r5gVY5yEarSf6ZXvkKn0ZK7MrAd1mreXyosbcxXhf2e80zrouk6RoYN85i8TMHDVwgac2PYQXrse2QiMRhF72USfS0DIphZs1vV2QRYagD5dhgpco7OmEaIHmvb9soymHnKmHJtOlD3snaqruB1WCyX9gy5p6sz/9DtWpwvK6ZoruhZ3SuTdcmOM8eCT9VIQJnADMZaIFtQZxxVVdrendaBheQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA8SdC/x5neyWC8Noraej1U8f1BS5ctIqo5K1Cx72+otXujB7KygRFDyaYGt4XKDM1Vnt+bBrKHPosbrbhDjuPzRJqlrq2HoR9ksckVWoN5Rgrgb15cH2SagUrz55vUn/7PmvENMSpBa8TTxKf6lMK43j80D5s0kzDCFREWGrcRa0V9OHmSOH+1Ux35W4xbbPpmYwODII81R+m2fM1VjgAsA0KxlElHVNFt8pO9+DTWU/j3qxsD6VhHDJ4hCBf6dra21bOi/6kckMMOHP5TPgLFlT88I/BqNqNFVIK65c1PJB8QS1b1jk6Ossy1FpVFiC0Fcub7PON7XizfgCV3nAn0 + + + + + + + + MIIC8DCCAdigAwIBAgIQH0zLxpk8ZIpL6FjEf3kvkTANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylNaWNyb3NvZnQgQXp1cmUgRmVkZXJhdGVkIFNTTyBDZXJ0aWZpY2F0ZTAeFw0yNDAzMjgwNzQ2NTlaFw0yNzAzMjgwNzQ2NDhaMDQxMjAwBgNVBAMTKU1pY3Jvc29mdCBBenVyZSBGZWRlcmF0ZWQgU1NPIENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArXs7OiS8m1ZM4y5yQv4Cze8uDkl44bI1zx3Pl+dQRcPHo7KmY8RKAzFgGCsxDyFqgzIPX9vTotBoqWFmj+U2FimzsFPV9GJVfJkmRbxG5NqKir8jlrziZwEe/3+8r5gVY5yEarSf6ZXvkKn0ZK7MrAd1mreXyosbcxXhf2e80zrouk6RoYN85i8TMHDVwgac2PYQXrse2QiMRhF72USfS0DIphZs1vV2QRYagD5dhgpco7OmEaIHmvb9soymHnKmHJtOlD3snaqruB1WCyX9gy5p6sz/9DtWpwvK6ZoruhZ3SuTdcmOM8eCT9VIQJnADMZaIFtQZxxVVdrendaBheQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA8SdC/x5neyWC8Noraej1U8f1BS5ctIqo5K1Cx72+otXujB7KygRFDyaYGt4XKDM1Vnt+bBrKHPosbrbhDjuPzRJqlrq2HoR9ksckVWoN5Rgrgb15cH2SagUrz55vUn/7PmvENMSpBa8TTxKf6lMK43j80D5s0kzDCFREWGrcRa0V9OHmSOH+1Ux35W4xbbPpmYwODII81R+m2fM1VjgAsA0KxlElHVNFt8pO9+DTWU/j3qxsD6VhHDJ4hCBf6dra21bOi/6kckMMOHP5TPgLFlT88I/BqNqNFVIK65c1PJB8QS1b1jk6Ossy1FpVFiC0Fcub7PON7XizfgCV3nAn0 + + + + + + Name + The mutable display name of the user. + + + Subject + An immutable, globally unique, non-reusable identifier of the user that is unique to the application for which a token is issued. + + + Given Name + First name of the user. + + + Surname + Last name of the user. + + + Display Name + Display name of the user. + + + Nick Name + Nick name of the user. + + + Authentication Instant + The time (UTC) when the user is authenticated to Windows Azure Active Directory. + + + Authentication Method + The method that Windows Azure Active Directory uses to authenticate users. + + + ObjectIdentifier + Primary identifier for the user in the directory. Immutable, globally unique, non-reusable. + + + TenantId + Identifier for the user's tenant. + + + IdentityProvider + Identity provider for the user. + + + Email + Email address of the user. + + + Groups + Groups of the user. + + + External Access Token + Access token issued by external identity provider. + + + External Access Token Expiration + UTC expiration time of access token issued by external identity provider. + + + External OpenID 2.0 Identifier + OpenID 2.0 identifier issued by external identity provider. + + + GroupsOverageClaim + Issued when number of user's group claims exceeds return limit. + + + Role Claim + Roles that the user or Service Principal is attached to + + + RoleTemplate Id Claim + Role template id of the Built-in Directory Roles that the user is a member of + + + + + https://login.microsoftonline.com/7b934664-cdcf-4e28-a3ee-1a5bcca0a1b6/wsfed + + + + + https://login.microsoftonline.com/7b934664-cdcf-4e28-a3ee-1a5bcca0a1b6/wsfed + + + + + + + + MIIC8DCCAdigAwIBAgIQH0zLxpk8ZIpL6FjEf3kvkTANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylNaWNyb3NvZnQgQXp1cmUgRmVkZXJhdGVkIFNTTyBDZXJ0aWZpY2F0ZTAeFw0yNDAzMjgwNzQ2NTlaFw0yNzAzMjgwNzQ2NDhaMDQxMjAwBgNVBAMTKU1pY3Jvc29mdCBBenVyZSBGZWRlcmF0ZWQgU1NPIENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArXs7OiS8m1ZM4y5yQv4Cze8uDkl44bI1zx3Pl+dQRcPHo7KmY8RKAzFgGCsxDyFqgzIPX9vTotBoqWFmj+U2FimzsFPV9GJVfJkmRbxG5NqKir8jlrziZwEe/3+8r5gVY5yEarSf6ZXvkKn0ZK7MrAd1mreXyosbcxXhf2e80zrouk6RoYN85i8TMHDVwgac2PYQXrse2QiMRhF72USfS0DIphZs1vV2QRYagD5dhgpco7OmEaIHmvb9soymHnKmHJtOlD3snaqruB1WCyX9gy5p6sz/9DtWpwvK6ZoruhZ3SuTdcmOM8eCT9VIQJnADMZaIFtQZxxVVdrendaBheQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA8SdC/x5neyWC8Noraej1U8f1BS5ctIqo5K1Cx72+otXujB7KygRFDyaYGt4XKDM1Vnt+bBrKHPosbrbhDjuPzRJqlrq2HoR9ksckVWoN5Rgrgb15cH2SagUrz55vUn/7PmvENMSpBa8TTxKf6lMK43j80D5s0kzDCFREWGrcRa0V9OHmSOH+1Ux35W4xbbPpmYwODII81R+m2fM1VjgAsA0KxlElHVNFt8pO9+DTWU/j3qxsD6VhHDJ4hCBf6dra21bOi/6kckMMOHP5TPgLFlT88I/BqNqNFVIK65c1PJB8QS1b1jk6Ossy1FpVFiC0Fcub7PON7XizfgCV3nAn0 + + + + + + https://sts.windows.net/7b934664-cdcf-4e28-a3ee-1a5bcca0a1b6/ + + + + + https://login.microsoftonline.com/7b934664-cdcf-4e28-a3ee-1a5bcca0a1b6/wsfed + + + + + https://login.microsoftonline.com/7b934664-cdcf-4e28-a3ee-1a5bcca0a1b6/wsfed + + + + + + + + MIIC8DCCAdigAwIBAgIQH0zLxpk8ZIpL6FjEf3kvkTANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylNaWNyb3NvZnQgQXp1cmUgRmVkZXJhdGVkIFNTTyBDZXJ0aWZpY2F0ZTAeFw0yNDAzMjgwNzQ2NTlaFw0yNzAzMjgwNzQ2NDhaMDQxMjAwBgNVBAMTKU1pY3Jvc29mdCBBenVyZSBGZWRlcmF0ZWQgU1NPIENlcnRpZmljYXRlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArXs7OiS8m1ZM4y5yQv4Cze8uDkl44bI1zx3Pl+dQRcPHo7KmY8RKAzFgGCsxDyFqgzIPX9vTotBoqWFmj+U2FimzsFPV9GJVfJkmRbxG5NqKir8jlrziZwEe/3+8r5gVY5yEarSf6ZXvkKn0ZK7MrAd1mreXyosbcxXhf2e80zrouk6RoYN85i8TMHDVwgac2PYQXrse2QiMRhF72USfS0DIphZs1vV2QRYagD5dhgpco7OmEaIHmvb9soymHnKmHJtOlD3snaqruB1WCyX9gy5p6sz/9DtWpwvK6ZoruhZ3SuTdcmOM8eCT9VIQJnADMZaIFtQZxxVVdrendaBheQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA8SdC/x5neyWC8Noraej1U8f1BS5ctIqo5K1Cx72+otXujB7KygRFDyaYGt4XKDM1Vnt+bBrKHPosbrbhDjuPzRJqlrq2HoR9ksckVWoN5Rgrgb15cH2SagUrz55vUn/7PmvENMSpBa8TTxKf6lMK43j80D5s0kzDCFREWGrcRa0V9OHmSOH+1Ux35W4xbbPpmYwODII81R+m2fM1VjgAsA0KxlElHVNFt8pO9+DTWU/j3qxsD6VhHDJ4hCBf6dra21bOi/6kckMMOHP5TPgLFlT88I/BqNqNFVIK65c1PJB8QS1b1jk6Ossy1FpVFiC0Fcub7PON7XizfgCV3nAn0 + + + + + + + + \ No newline at end of file diff --git a/ckan/saml/dbca_staging_sp.xml b/ckan/saml/dbca_staging_sp.xml new file mode 100644 index 00000000..8b5ef7e8 --- /dev/null +++ b/ckan/saml/dbca_staging_sp.xml @@ -0,0 +1,7 @@ + + + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + + + + diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index 5486afb6..91bd0b64 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -58,6 +58,8 @@ pip3 install -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckane # DOI # pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi +# SAML2 +pip3 install -e git+https://github.com/keitaroinc/ckanext-saml2auth.git@v1.3.0#egg=ckanext-saml2auth ## DBCA Project ## diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 559e0a47..bbf4edb9 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -34,7 +34,6 @@ services: - ./src:/srv/app/src_extensions - ./ckan/config:/srv/app/config - ckan_logs:/srv/app/logs - - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] @@ -66,6 +65,7 @@ services: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions - ckan_logs:/srv/app/logs + - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/docker-compose.yml b/docker-compose.yml index b9be683b..ad5ffc5c 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,7 @@ services: volumes: - ckan_storage:/var/lib/ckan - ckan_logs:/srv/app/logs + - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] From c6622ce74e1a1afec4794afceeea2ba185fde8f9 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 4 Apr 2024 10:47:02 +1300 Subject: [PATCH 063/123] Fix SAML idp_metadata local path in dbca.ini --- ckan/config/dbca.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 10153a66..f6e916cf 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -79,7 +79,7 @@ ckanext.saml2auth.idp_metadata.location = local # Path to a local file accessible on the server the service runs on # Ignore this config if the idp metadata location is set to: remote -ckanext.saml2auth.idp_metadata.local_path = /srv/app/saml/dbca_idp.xmlå +ckanext.saml2auth.idp_metadata.local_path = /srv/app/saml/dbca_idp.xml # Corresponding SAML user field for firstname ckanext.saml2auth.user_firstname = givenname From 3a4e4c04032b2162ba6e8af1a214074276102823 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 4 Apr 2024 10:51:32 +1300 Subject: [PATCH 064/123] Update database import process --- .ahoy.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.ahoy.yml b/.ahoy.yml index 66cf740b..98b1c7b3 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -124,9 +124,20 @@ commands: usage: Pipe in a postgres dump file. `ahoy db-import local.dump` cmd: | if [ -e "$@" ] ; then + if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then + CKAN_CONTAINER_NAME=ckan-dev + fi + SERVICE=$CKAN_CONTAINER_NAME + # Clean the database + ahoy run "ckan db clean --yes" + # Import the database docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@" + # Initialise plugin tables + docker compose -f $DOCKER_COMPOSE exec -T $CKAN_CONTAINER_NAME sh -c 'export CKAN__PLUGINS=$(grep "^ckan\.plugins" $APP_DIR/config/dbca.ini | cut -d"=" -f2) && bash /docker-entrypoint.d/02_setup_dbca.sh' + # Rebuild the Solr index ahoy reindex-solr - else echo "Provided sql file" "$@" "does not exist" + else + echo "Provided sql file" "$@" "does not exist" fi import-db: From e70d1952c1342917a02875a7caf1a8ddfdc55b58 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 5 Apr 2024 14:50:56 +1300 Subject: [PATCH 065/123] Update to use dbca.ini config --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 7301369b..f3c32fd8 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -16,7 +16,7 @@ if [[ $CKAN__PLUGINS == *"xloader"* ]]; then CKAN_INI=$APP_DIR/ckan.ini # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) echo "Setting a temporary value for ckanext.xloader.api_token" - ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" + ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $APP_DIR/config/dbca.ini user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" fi CKAN_INI=$APP_DIR/config/dbca.ini From 382d94305ba8f7f10bde9854268beeaf43cb6404 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 5 Apr 2024 14:53:48 +1300 Subject: [PATCH 066/123] Add datatables_view plugin to CKAN worker Dockerfile --- ckan/Dockerfile.worker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 2ee7d549..757b9588 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -2,7 +2,7 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} # Override the default CKAN config plugins to only use the ones we need for the worker -ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi envvars" +ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi datatables_view envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" # Update the CKAN config log file handler to use the worker log file RUN ckan config-tool $CKAN_INI -s handler_fileHandler "args = ('/srv/app/logs/ckan-worker.log', 'midnight', 1, 7)" From 61676e0ad59f2cb94ca70ed9f73315b4984e60f5 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 5 Apr 2024 15:35:47 +1300 Subject: [PATCH 067/123] Update SAML configuration in dbca.ini --- ckan/config/dbca.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index f6e916cf..b83098f8 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -104,6 +104,11 @@ ckanext.saml2auth.enable_ckan_internal_login = True # Define the entity ID. Default is urn:mace:umu.se:saml:ckan:sp ckanext.saml2auth.entity_id = urn:mace:umu.se:saml:ckan_dbca_staging:sp +# Signed responses and assertions +ckanext.saml2auth.want_response_signed = false +ckanext.saml2auth.want_assertions_signed = False +ckanext.saml2auth.want_assertions_or_response_signed = False + ## Logging configuration [loggers] From 8235a593bb613e6479c411e85bb4433e802f9ccc Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Mon, 8 Apr 2024 11:48:42 +0800 Subject: [PATCH 068/123] Added saml config for NameIDFormat config Set saml package logging to debug --- ckan/config/dbca.ini | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index b83098f8..d9ee58f4 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -104,6 +104,10 @@ ckanext.saml2auth.enable_ckan_internal_login = True # Define the entity ID. Default is urn:mace:umu.se:saml:ckan:sp ckanext.saml2auth.entity_id = urn:mace:umu.se:saml:ckan_dbca_staging:sp +# A list of string values that will be used to set the element of the metadata of an entity. +# Default: urn:oasis:names:tc:SAML:2.0:nameid-format:persistent +ckanext.saml2auth.sp.name_id_format = urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress + # Signed responses and assertions ckanext.saml2auth.want_response_signed = false ckanext.saml2auth.want_assertions_signed = False @@ -112,7 +116,7 @@ ckanext.saml2auth.want_assertions_or_response_signed = False ## Logging configuration [loggers] -keys = root, ckan, ckanext, werkzeug +keys = root, ckan, ckanext, werkzeug, saml2 [handlers] keys = console, fileHandler @@ -142,6 +146,12 @@ handlers = console, fileHandler qualname = ckanext propagate = 0 +[logger_saml2] +level = DEBUG +handlers = console, fileHandler +qualname = saml2 +propagate = 0 + [handler_console] class = StreamHandler args = (sys.stderr,) From cfe267e980c50952d5139b7b747b2a6028062fe6 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 9 Apr 2024 07:57:32 +0800 Subject: [PATCH 069/123] Updated saml2auth config key metadata --- ckan/config/dbca.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index d9ee58f4..3e4e55c7 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -82,7 +82,7 @@ ckanext.saml2auth.idp_metadata.location = local ckanext.saml2auth.idp_metadata.local_path = /srv/app/saml/dbca_idp.xml # Corresponding SAML user field for firstname -ckanext.saml2auth.user_firstname = givenname +ckanext.saml2auth.user_firstname = givenName # Corresponding SAML user field for lastname ckanext.saml2auth.user_lastname = surname @@ -92,7 +92,7 @@ ckanext.saml2auth.user_lastname = surname #ckanext.saml2auth.user_fullname = fullname # Corresponding SAML user field for email -ckanext.saml2auth.user_email = emailaddress +ckanext.saml2auth.user_email = emailAddress # Optional: Corresponding SAML user field for groups From ff878a51c60e82ed90e4fe61e4173431086f16f8 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 9 Apr 2024 12:41:47 +0800 Subject: [PATCH 070/123] Update SAML configuration for logout requests --- ckan/config/dbca.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 3e4e55c7..e2104f55 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -113,6 +113,13 @@ ckanext.saml2auth.want_response_signed = false ckanext.saml2auth.want_assertions_signed = False ckanext.saml2auth.want_assertions_or_response_signed = False +# Indicates if this entity will sign the Logout Requests originated from it +ckanext.saml2auth.logout_requests_signed = False + +# Saml logout request preferred binding settings variable +# Default: urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST +ckanext.saml2auth.logout_expected_binding = urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect + ## Logging configuration [loggers] From f009bb7192c3a96cccffbd3f9975e4d242cce082 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 19 Apr 2024 08:37:00 +0800 Subject: [PATCH 071/123] Solr queries not working due to local parameters not being allowed since CKAN 2.10.4 --- ckan/config/dbca.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index e2104f55..29573a48 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -29,6 +29,10 @@ ckan.auth.user_delete_organizations = false ## Activity Streams Settings ################################################### ckan.activity_streams_email_notifications = true +## Search Settings ############################################################# +# ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 +ckan.search.solr_allowed_query_parsers = frange + ## CKAN Extensions configuration ############################################### From 445d52555de0567d7b9f40e53ba2621ec2494598 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 22 Apr 2024 14:11:26 -0400 Subject: [PATCH 072/123] remove container names --- .env.example | 9 --------- docker-compose.yml | 6 ------ 2 files changed, 15 deletions(-) diff --git a/.env.example b/.env.example index 3508b6e7..b5760c2b 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,3 @@ -# Container names -NGINX_CONTAINER_NAME=nginx -REDIS_CONTAINER_NAME=redis -POSTGRESQL_CONTAINER_NAME=db -SOLR_CONTAINER_NAME=solr -DATAPUSHER_CONTAINER_NAME=datapusher -CKAN_CONTAINER_NAME=ckan -WORKER_CONTAINER_NAME=ckan-worker - # Host Ports CKAN_PORT_HOST=5000 NGINX_PORT_HOST=81 diff --git a/docker-compose.yml b/docker-compose.yml index 0f5330fb..4f004d71 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ volumes: services: nginx: - container_name: ${NGINX_CONTAINER_NAME} build: context: nginx/ dockerfile: Dockerfile @@ -23,7 +22,6 @@ services: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" ckan: - container_name: ${CKAN_CONTAINER_NAME} build: context: ckan/ dockerfile: Dockerfile @@ -50,7 +48,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] datapusher: - container_name: ${DATAPUSHER_CONTAINER_NAME} networks: - ckannet - dbnet @@ -60,7 +57,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"] db: - container_name: ${POSTGRESQL_CONTAINER_NAME} build: context: postgresql/ networks: @@ -82,7 +78,6 @@ services: test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"] solr: - container_name: ${SOLR_CONTAINER_NAME} networks: - solrnet image: ckan/ckan-solr:${SOLR_IMAGE_VERSION} @@ -93,7 +88,6 @@ services: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"] redis: - container_name: ${REDIS_CONTAINER_NAME} image: redis:${REDIS_VERSION} networks: - redisnet From b78b077d32dcdc8fafbc7d7cc80dc735bbd1f323 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 23 Apr 2024 11:58:27 +0800 Subject: [PATCH 073/123] Updated uwsgi user to root user for testing --- ckan/config/uwsgi.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/uwsgi.ini b/ckan/config/uwsgi.ini index b60f71e8..d2705584 100644 --- a/ckan/config/uwsgi.ini +++ b/ckan/config/uwsgi.ini @@ -4,8 +4,8 @@ plugins= http,python socket=/tmp/uwsgi.sock wsgi-file=/srv/app/wsgi.py module=wsgi:application -uid=92 -gid=92 +uid=root +gid=root http=0.0.0.0:5000 master=enable-threads lazy-apps=true From 32af5cfbf8d1c81fd7e309d158117445067a8f13 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 16:29:18 -0400 Subject: [PATCH 074/123] fix readme toc, headings, replace one screenshot of text --- README.md | 76 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 5f689cc9..79cdceb3 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,25 @@ # Docker Compose setup for CKAN -* [Overview](#overview) -* [Installing Docker](#installing-docker) -* [docker compose vs docker-compose](#docker-compose-vs-docker-compose) -* [Install CKAN plus dependencies](#install-ckan-plus-dependencies) -* [Development mode](#development-mode) - * [Create an extension](#create-an-extension) - * [Running HTTPS on development mode](#running-https-on-development-mode) -* [CKAN images](#ckan-images) - * [Extending the base images](#extending-the-base-images) - * [Applying patches](#applying-patches) -* [Debugging with pdb](#pdb) -* [Datastore and Datapusher](#Datastore-and-datapusher) -* [NGINX](#nginx) -* [The ckanext-envvars extension](#envvars) -* [The CKAN_SITE_URL parameter](#CKAN_SITE_URL) -* [Changing the base image](#Changing-the-base-image) -* [Replacing DataPusher with XLoader](#Replacing-DataPusher-with-XLoader) +* [1. Overview](#1--overview) +* [2. Installing Docker](#2--installing-docker) +* [3. docker compose vs docker-compose](#3--docker-compose-vs-docker-compose) +* [4. Install (build and run) CKAN plus dependencies](#4--install-build-and-run-ckan-plus-dependencies) + * [Base mode](#base-mode) + * [Development mode](#development-mode) + * [Create an extension](#create-an-extension) + * [Running HTTPS on development mode](#running-https-on-development-mode) +* [5. CKAN images](#5--ckan-images) + * [Extending the base images](#extending-the-base-images) + * [Applying patches](#applying-patches) +* [6. Debugging with pdb](#6--debugging-with-pdb) +* [7. Datastore and Datapusher](#7--datastore-and-datapusher) +* [8. NGINX](#8--nginx) +* [9. ckanext-envvars](#9--ckanext-envvars) +* [10. CKAN_SITE_URL](#10--CKAN_SITE_URL) +* [11. Manage new users](#11-manage-new-users) +* [12. Changing the base image](#12--changing-the-base-image) +* [13. Replacing DataPusher with XLoader](#13--replacing-datapusher-with-xLoader) ## 1. Overview @@ -51,14 +53,14 @@ more information. ## 4. Install (build and run) CKAN plus dependencies -#### Base mode +### Base mode Use this if you are a maintainer and will not be making code changes to CKAN or to CKAN extensions Copy the included `.env.example` and rename it to `.env`. Modify it depending on your own needs. Please note that when accessing CKAN directly (via a browser) ie: not going through NGINX you will need to make sure you have "ckan" set up -to be an alias to localhost in the local hosts file. Either that or you will need to change the `.env` entry for CKAN_SITE_URL +to be an alias to localhost in the local hosts file. Either that or you will need to change the `.env` entry for `CKAN_SITE_URL` Using the default values on the `.env.example` file will get you a working CKAN instance. There is a sysadmin user created by default with the values defined in `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD`(`ckan_admin` and `test1234` by default). This should be obviously changed before running this setup as a public CKAN instance. @@ -81,7 +83,7 @@ At the end of the container start sequence there should be 6 containers running After this step, CKAN should be running at `CKAN_SITE_URL`. -#### Development mode +### Development mode Use this mode if you are making code changes to CKAN and either creating new extensions or making code changes to existing extensions. This mode also uses the `.env` file for config options. @@ -95,10 +97,10 @@ To start the containers: docker compose -f docker-compose.dev.yml up -See [CKAN Images](#ckan-images) for more details of what happens when using development mode. +See [CKAN Images](#5--ckan-images) for more details of what happens when using development mode. -##### Create an extension +#### Create an extension You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tutorial.html#creating-a-new-extension) instructions to create a CKAN extension, only executing the command inside the CKAN container and setting the mounted `src/` folder as output: @@ -109,7 +111,7 @@ You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tuto The new extension files and directories are created in the `/srv/app/src_extensions/` folder in the running container. They will also exist in the local src/ directory as local `/src` directory is mounted as `/srv/app/src_extensions/` on the ckan container. You might need to change the owner of its folder to have the appropiate permissions. -##### Running HTTPS on development mode +#### Running HTTPS on development mode Sometimes is useful to run your local development instance under HTTPS, for instance if you are using authentication extensions like [ckanext-saml2auth](https://github.com/keitaroinc/ckanext-saml2auth). To enable it, set the following in your `.env` file: @@ -138,7 +140,7 @@ The Docker image config files used to build your CKAN project are located in the * Any custom changes to the scripts run during container start up can be made to scripts in the `setup/` directory. For instance if you wanted to change the port on which CKAN runs you would need to make changes to the Docker Compose yaml file, and the `start_ckan.sh.override` file. Then you would need to add the following line to the Dockerfile ie: `COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh`. The `start_ckan.sh` file in the locally built image would override the `start_ckan.sh` file included in the base image -## 6. Extending the base images +### Extending the base images You can modify the docker files to build your own customized image tailored to your project, installing any extensions and extra requirements needed. For example here is where you would update to use a different CKAN base image ie: `ckan/ckan-base:` @@ -177,7 +179,7 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/ NB: There are a number of extension examples commented out in the Dockerfile.dev file -## 7. Applying patches +### Applying patches When building your project specific CKAN images (the ones defined in the `ckan/` folder), you can apply patches to CKAN core or any of the built extensions. To do so create a folder inside `ckan/patches` with the name of the @@ -201,22 +203,28 @@ ckan ``` -## 8. pdb +## 6. Debugging with pdb Add these lines to the `ckan-dev` service in the docker-compose.dev.yml file -![pdb](https://user-images.githubusercontent.com/54408245/179964232-9e98a451-5fe9-4842-ba9b-751bcc627730.png) +```yaml +ports: + - "0.0.0.0:${CKAN_PORT}:5000" + +stdin_open: true +tty: true +``` Debug with pdb (example) - Interact with `docker attach $(docker container ls -qf name=ckan)` command: `python -m pdb /usr/lib/ckan/venv/bin/ckan --config /srv/app/ckan.ini run --host 0.0.0.0 --passthrough-errors` -## 9. Datastore and datapusher +## 7. Datastore and datapusher The Datastore database and user is created as part of the entrypoint scripts for the db container. There is also a Datapusher container running the latest version of Datapusher. -## 10. NGINX +## 8. NGINX The base Docker Compose configuration uses an NGINX image as the front-end (ie: reverse proxy). It includes HTTPS running on port number 8443. A "self-signed" SSL certificate is generated as part of the ENTRYPOINT. The NGINX `server_name` directive and the `CN` field in the SSL certificate have been both set to 'localhost'. This should obviously not be used for production. @@ -224,7 +232,7 @@ Creating the SSL cert and key files as follows: `openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/C=DE/ST=Berlin/L=Berlin/O=None/CN=localhost" -keyout ckan-local.key -out ckan-local.crt` The `ckan-local.*` files will then need to be moved into the nginx/setup/ directory -## 11. envvars +## 9. ckanext-envvars The ckanext-envvars extension is used in the CKAN Docker base repo to build the base images. This extension checks for environmental variables conforming to an expected format and updates the corresponding CKAN config settings with its value. @@ -245,11 +253,11 @@ These parameters can be added to the `.env` file For more information please see [ckanext-envvars](https://github.com/okfn/ckanext-envvars) -## 12. CKAN_SITE_URL +## 10. CKAN_SITE_URL For convenience the CKAN_SITE_URL parameter should be set in the .env file. For development it can be set to http://localhost:5000 and non-development set to https://localhost:8443 -## 13. Manage new users +## 11. Manage new users 1. Create a new user from the Docker host, for example to create a new user called 'admin' @@ -267,12 +275,12 @@ For convenience the CKAN_SITE_URL parameter should be set in the .env file. For `ckan -c ckan.ini user remove admin` -## 14. Changing the base image +## 12. Changing the base image The base image used in the CKAN Dockerfile and Dockerfile.dev can be changed so a different DockerHub image is used eg: ckan/ckan-base:2.9.9 could be used instead of ckan/ckan-base:2.10.1 -## 15. Replacing DataPusher with XLoader +## 13. Replacing DataPusher with XLoader Check out the wiki page for this: https://github.com/ckan/ckan-docker/wiki/Replacing-DataPusher-with-XLoader From 5c550f4cd1503d2ff1221466169f8f0b24c47ef3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 16:34:00 -0400 Subject: [PATCH 075/123] now only one dash --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 79cdceb3..7eecac39 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,25 @@ # Docker Compose setup for CKAN -* [1. Overview](#1--overview) -* [2. Installing Docker](#2--installing-docker) -* [3. docker compose vs docker-compose](#3--docker-compose-vs-docker-compose) -* [4. Install (build and run) CKAN plus dependencies](#4--install-build-and-run-ckan-plus-dependencies) +* [1. Overview](#1-overview) +* [2. Installing Docker](#2-installing-docker) +* [3. docker compose vs docker-compose](#3-docker-compose-vs-docker-compose) +* [4. Install (build and run) CKAN plus dependencies](#4-install-build-and-run-ckan-plus-dependencies) * [Base mode](#base-mode) * [Development mode](#development-mode) * [Create an extension](#create-an-extension) * [Running HTTPS on development mode](#running-https-on-development-mode) -* [5. CKAN images](#5--ckan-images) +* [5. CKAN images](#5-ckan-images) * [Extending the base images](#extending-the-base-images) * [Applying patches](#applying-patches) -* [6. Debugging with pdb](#6--debugging-with-pdb) -* [7. Datastore and Datapusher](#7--datastore-and-datapusher) -* [8. NGINX](#8--nginx) -* [9. ckanext-envvars](#9--ckanext-envvars) -* [10. CKAN_SITE_URL](#10--CKAN_SITE_URL) +* [6. Debugging with pdb](#6-debugging-with-pdb) +* [7. Datastore and Datapusher](#7-datastore-and-datapusher) +* [8. NGINX](#8-nginx) +* [9. ckanext-envvars](#9-ckanext-envvars) +* [10. CKAN_SITE_URL](#10-CKAN_SITE_URL) * [11. Manage new users](#11-manage-new-users) -* [12. Changing the base image](#12--changing-the-base-image) -* [13. Replacing DataPusher with XLoader](#13--replacing-datapusher-with-xLoader) +* [12. Changing the base image](#12-changing-the-base-image) +* [13. Replacing DataPusher with XLoader](#13-replacing-datapusher-with-xLoader) ## 1. Overview From 72eade8ccad2eb2275af5f2bf52ed5ed74dc1b44 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 16:40:11 -0400 Subject: [PATCH 076/123] link to base images repo --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7eecac39..da7b817a 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ To start the containers: docker compose -f docker-compose.dev.yml up -See [CKAN Images](#5--ckan-images) for more details of what happens when using development mode. +See [CKAN images](#5-ckan-images) for more details of what happens when using development mode. #### Create an extension @@ -142,6 +142,8 @@ The Docker image config files used to build your CKAN project are located in the ### Extending the base images +The CKAN base images are built from https://github.com/ckan/ckan-docker-base/ + You can modify the docker files to build your own customized image tailored to your project, installing any extensions and extra requirements needed. For example here is where you would update to use a different CKAN base image ie: `ckan/ckan-base:` To perform extra initialization steps you can add scripts to your custom images and copy them to the `/docker-entrypoint.d` folder (The folder should be created for you when you build the image). Any `*.sh` and `*.py` file in that folder will be executed just after the main initialization script ([`prerun.py`](https://github.com/ckan/ckan-docker-base/blob/main/ckan-2.9/base/setup/prerun.py)) is executed and just before the web server and supervisor processes are started. From f4e152855fc6f32e743457d5f9ff1c49be3ca083 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 17:16:59 -0400 Subject: [PATCH 077/123] compose version is no longer used https://github.com/compose-spec/compose-spec/blob/master/spec.md#version-and-name-top-level-elements --- docker-compose.dev.yml | 2 -- docker-compose.yml | 3 --- 2 files changed, 5 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dc2ecdf9..e41d0ee5 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: "3" - volumes: ckan_storage: pg_data: diff --git a/docker-compose.yml b/docker-compose.yml index 4f004d71..d37c8efd 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,3 @@ -version: "3" - - volumes: ckan_storage: pg_data: From eee098b9280c97961f786da4e459c1ea323cb181 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 17:25:17 -0400 Subject: [PATCH 078/123] default datatables plugin instead of recline --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index b5760c2b..830cf949 100644 --- a/.env.example +++ b/.env.example @@ -69,7 +69,7 @@ NGINX_PORT=80 NGINX_SSLPORT=443 # Extensions -CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher" +CKAN__PLUGINS="envvars image_view text_view datatables_view datastore datapusher" CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__PORT=6379 From d7caf83eac0d1ddf1fd1274e547a600bdfcf7d28 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 24 Apr 2024 10:18:44 +1200 Subject: [PATCH 079/123] Revert "Updated uwsgi user to root user for testing" This reverts commit b78b077d32dcdc8fafbc7d7cc80dc735bbd1f323. --- ckan/config/uwsgi.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/uwsgi.ini b/ckan/config/uwsgi.ini index d2705584..b60f71e8 100644 --- a/ckan/config/uwsgi.ini +++ b/ckan/config/uwsgi.ini @@ -4,8 +4,8 @@ plugins= http,python socket=/tmp/uwsgi.sock wsgi-file=/srv/app/wsgi.py module=wsgi:application -uid=root -gid=root +uid=92 +gid=92 http=0.0.0.0:5000 master=enable-threads lazy-apps=true From dddf4434a7ee41496c3b49576ea10d72580ca26b Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 24 Apr 2024 10:28:04 +1200 Subject: [PATCH 080/123] Run the uwsgi command from the ckan user --- ckan/setup/dbca_start_ckan.sh.override | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index d9e3bbff..3b9523bb 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -40,7 +40,7 @@ then # Start supervisord supervisord --configuration /etc/supervisord.conf & # Start uwsgi - uwsgi -i $APP_DIR/config/uwsgi.ini + su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' else echo "[prerun] failed...not starting CKAN." fi \ No newline at end of file From 726d586ffbd7c3ab5ccdde24a2b9abd7fad14753 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 23 Apr 2024 18:39:45 -0400 Subject: [PATCH 081/123] start dev ckan in parallel --- docker-compose.dev.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dc2ecdf9..7f526ed6 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -15,13 +15,6 @@ services: - TZ=${TZ} env_file: - .env - depends_on: - db: - condition: service_healthy - solr: - condition: service_healthy - redis: - condition: service_healthy ports: - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" volumes: From f69307e1d713a532929cba833c50518cafbea584 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 24 Apr 2024 13:16:24 +1200 Subject: [PATCH 082/123] Add cron job to rebuild solr index --- ckan/setup/dbca_ckan_cron_jobs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index c25ee32b..a3a4f064 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,4 +1,6 @@ # Crontab for CKAN cron jobs +# Hourly Rebuild Solr Index +0 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini ckan search-index # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # 8am task to schedule embargo datasets to public visibility From fcd9648e699e88f62fed0204c646afc988c006d3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Wed, 24 Apr 2024 16:59:01 -0400 Subject: [PATCH 083/123] volumes for installed files for faster startup --- docker-compose.dev.yml | 2 ++ docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dc2ecdf9..7d3831aa 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,6 +27,8 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions + - /usr/lib/python3.10/site-packages + - /root/.vscode-server restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/docker-compose.yml b/docker-compose.yml index 4f004d71..d241a830 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: condition: service_healthy volumes: - ckan_storage:/var/lib/ckan + - /usr/lib/python3.10/site-packages restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] From 275a9374a7f152b79babe6b7f27709d10fac1d8a Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Wed, 24 Apr 2024 17:34:38 -0400 Subject: [PATCH 084/123] document debugpy option --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index da7b817a..7e4fde87 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ * [Development mode](#development-mode) * [Create an extension](#create-an-extension) * [Running HTTPS on development mode](#running-https-on-development-mode) + * [Remote Debugging with VS Code](#remote-debugging-with-vs-code) * [5. CKAN images](#5-ckan-images) * [Extending the base images](#extending-the-base-images) * [Applying patches](#applying-patches) @@ -115,15 +116,42 @@ The new extension files and directories are created in the `/srv/app/src_extensi Sometimes is useful to run your local development instance under HTTPS, for instance if you are using authentication extensions like [ckanext-saml2auth](https://github.com/keitaroinc/ckanext-saml2auth). To enable it, set the following in your `.env` file: +``` USE_HTTPS_FOR_DEV=true +``` and update the site URL setting: +``` CKAN_SITE_URL=https://localhost:5000 +``` After recreating the `ckan-dev` container, you should be able to access CKAN at https://localhost:5000 +#### Remote Debugging with VS Code + +[Visual Studio Code](https://code.visualstudio.com/) is a free IDE that includes remote +debugging for Python applications. To debug CKAN you must enable `debugpy` for your +development instance in your `.env` file: + +``` + USE_DEBUGPY_FOR_DEV=true +``` + +Then start the containers in [development mode](#development-mode) and launch VS Code. + +In VS Code: + +1. Install the "Dev Container" extension: press CTRL+SHIFT+X, type "dev container", click "install" +2. Click the "Open a Remote Window" button in the bottom-left of the VS Code window +3. Click "Attach to Running Container..." and select your ckan-dev container, e.g. "ckan-docker-ckan-dev-1" +4. Click the "Run and Debug" icon on the left panel then "create a launch.json", select "Python Debugger", "Remote Attach", host "localhost" and port "5678" +5. Press F5 or click the "Run" menu and "Start Debugging" + +You can now set breakpoints and remote debug your CKAN development instance. + + ## 5. CKAN images ![ckan images](https://user-images.githubusercontent.com/54408245/207079416-a01235af-2dea-4425-b6fd-f8c3687dd993.png) From 4f751b7e72e7682d4fb96b3020831ae2d47f23e3 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 29 Apr 2024 13:43:50 -0400 Subject: [PATCH 085/123] ckan-dev needs db, solr, redis --- docker-compose.dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 48a08e73..aa97df6d 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -13,6 +13,10 @@ services: - TZ=${TZ} env_file: - .env + links: + - db + - solr + - redis ports: - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" volumes: From bbcadaf920823fa04f79e356d845272f5741bd44 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 29 Apr 2024 14:33:47 -0400 Subject: [PATCH 086/123] volume for pip cache, keep volumes --- docker-compose.dev.yml | 10 +++++++--- docker-compose.yml | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index aa97df6d..b1a56a49 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,6 +2,9 @@ volumes: ckan_storage: pg_data: solr_data: + pip_cache: + site_packages: + vscode_server: services: @@ -22,12 +25,13 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions - - /usr/lib/python3.10/site-packages - - /root/.vscode-server + - pip_cache:/root/.cache/pip + - site_packages:/usr/lib/python3.10/site-packages + - vscode_server:/root/.vscode-server restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] - + datapusher: image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION} restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 97ab43f4..e7c61970 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,8 @@ volumes: ckan_storage: pg_data: solr_data: + pip_cache: + site_packages: services: @@ -40,7 +42,8 @@ services: condition: service_healthy volumes: - ckan_storage:/var/lib/ckan - - /usr/lib/python3.10/site-packages + - pip_cache:/root/.cache/pip + - site_packages:/usr/lib/python3.10/site-packages restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] From 587f9e14c81c3a6d08427f31561d1af45f21a664 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 30 Apr 2024 09:46:24 +0800 Subject: [PATCH 087/123] Update README.MD with instructions for deleting and reloading spatial data --- spatial_data/README.MD | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/spatial_data/README.MD b/spatial_data/README.MD index ed982eb8..069767f7 100644 --- a/spatial_data/README.MD +++ b/spatial_data/README.MD @@ -4,7 +4,7 @@ This directory is used to store GeoJSON files that will be loaded from docker en ## Usage -To add a new GeoJSON file, simply place it in this directory. The container will automatically load the file if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. +To add a new GeoJSON file, simply place it in this directory. The cakn-worker container will automatically load any files on deployment if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. Please ensure that the GeoJSON file follows the correct format and contains valid spatial data. @@ -26,3 +26,16 @@ Example: "tenure.geojson": {"layer": "DBCA Managed Tenure", "code":"LEG_TENURE", "name":"LEG_NAME"} } ``` + +## Delete all spatial data and reload geojson files + + This is recommended any time there are updates to any of the geojson files + + Truncate database table dbca_spatial + ``` + TRUNCATE "public"."dbca_spatial"; + ``` + ReDeploy ckan-worker container or run CLI command in container ckan-worker to load spatial data + ``` + ckan -c $CKAN_INI dbca load_spatial_data + ``` \ No newline at end of file From 610279b50ecc3daa9681fe4234c586d9113e480c Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 30 Apr 2024 09:47:04 +0800 Subject: [PATCH 088/123] Update view config --- ckan/config/dbca.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 29573a48..424f0609 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report resource_proxy geo_view shp_view officedocs_view envvars +ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view resource_proxy datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = text_view datatables_view pdf_view geo_view shp_view officedocs_view +ckan.views.default_views = image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU From 8457aa829bb5a864b2bdc3a306793aae8bf61d28 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 30 Apr 2024 09:53:51 +0800 Subject: [PATCH 089/123] Fixed typo --- spatial_data/README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spatial_data/README.MD b/spatial_data/README.MD index 069767f7..e341e917 100644 --- a/spatial_data/README.MD +++ b/spatial_data/README.MD @@ -4,7 +4,7 @@ This directory is used to store GeoJSON files that will be loaded from docker en ## Usage -To add a new GeoJSON file, simply place it in this directory. The cakn-worker container will automatically load any files on deployment if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. +To add a new GeoJSON file, simply place it in this directory. The ckan-worker container will automatically load any files on deployment if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. Please ensure that the GeoJSON file follows the correct format and contains valid spatial data. From 5792a24b7955dc87e01384bdf66267ff4ec81367 Mon Sep 17 00:00:00 2001 From: Ben Richardson <1963681+ben3000@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:57:00 +0800 Subject: [PATCH 090/123] Adds file creation instructions. --- spatial_data/README.MD | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/spatial_data/README.MD b/spatial_data/README.MD index e341e917..9f5aee6a 100644 --- a/spatial_data/README.MD +++ b/spatial_data/README.MD @@ -2,6 +2,37 @@ This directory is used to store GeoJSON files that will be loaded from docker entrypoint file `02_setup_dbca.sh` using the CLI command `ckan -c $CKAN_INI dbca load_spatial_data` if the mapping exists in `ckanext.dbca.spatial_data_mapping`. +## Creating the data files + +As each base layer can have many rows per named feature, it is important that we convert this to a dataset that contains only one row for each named feature. The base layers can have many rows because some features are naturally separate from one another, such as the polygon for an island in the same named feature as that on the mainland. + +| Layer Name | Agency Layer Name | Property for Label | Property for Code | Output Filename | +|------------------------|-----------------------------------|--------------------|-------------------|-------------------------| +| IBRA Regions | IBRA Vegetation (Subregions) - WA | IWA_REG_NAME_7 | IWA_REG_CODE_7 | ibra-regions.geojson | +| IBRA Subregions | IBRA Vegetation (Subregions) - WA | IWA_SUB_NAME_7 | IWA_SUB_CODE_7 | ibra-subregions.geojson | +| IMCRA Regions | IMCRA Regions (Australia) | MESO_NAME | MESO_ABBR | imcra-regions.geojson | +| Local Government Areas | Local Govt. Authorities (LGA) | LGA_LGA_NAME | LGA_TYPE | lga-wa.geojson | +| DBCA Managed Tenure | Managed Tenure | LEG_NAME | LEG_TENURE | tenure.geojson | + +For each layer: + +1. Firstly, we must convert the layer into a single row for each unique label-polygon instance. We accomplish this by using the “Dissolve” command in QGIS. + 1. With the new temporary layer selected, choose Vector > Geoprocessing Tools > Dissolve… from the menu. + 2. Select the layer property for the label for the newly created temporary layer (see the table above) to tell QGIS which attribute to “unique” on. + 3. Run the command. + +2. Secondly, we must simplify each polygon in a layer so it takes up less than 1 MB per polygon. We accomplish this by using the “Simplify” command in QGIS. + 1. Open one of the target layers (above) in QGIS, then select Vector > Geometry Tools > Simplify… from the menu. + 2. While the layer is selected in the Layers window, choose Distance (Douglas-Peucker) from the Simplification method dropdown and use a tolerance of 0.05. Leave the other options as they are to generate a temporary layer. + 3. Run the command. + +3. Lastly, export the resulting layer to GeoJSON. + + 1. Right-click the layer in the Layer window and choose Export > Save Features As… from the popup menu. + 2. Give the file a name, e.g. ibra-subregions.geojson matching the above name in the Output Filename column. + 3. Run the command. + 4. Archive the file to zip, e.g. ibra-subregions.zip. + ## Usage To add a new GeoJSON file, simply place it in this directory. The ckan-worker container will automatically load any files on deployment if there is a corresponding mapping defined in `ckanext.dbca.spatial_data_mapping`. @@ -38,4 +69,4 @@ Example: ReDeploy ckan-worker container or run CLI command in container ckan-worker to load spatial data ``` ckan -c $CKAN_INI dbca load_spatial_data - ``` \ No newline at end of file + ``` From c1968b5a625860aab845ab8abd1beb0eb184e725 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 30 Apr 2024 16:15:34 +1200 Subject: [PATCH 091/123] Updated supervisor to be run from CKAN user --- ckan/Dockerfile.worker | 3 ++- ckan/setup/dbca_start_ckan.sh.override | 2 +- .../dbca_start_ckan_development.sh.override | 2 +- ckan/supervisor/ckan_cron_jobs.conf | 2 +- ckan/supervisor/supervisord.conf | 23 +++++++++++++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 ckan/supervisor/supervisord.conf diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 757b9588..5e11554d 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -8,7 +8,8 @@ RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" RUN ckan config-tool $CKAN_INI -s handler_fileHandler "args = ('/srv/app/logs/ckan-worker.log', 'midnight', 1, 7)" ## Supervisor config -COPY supervisor/*.conf /etc/supervisord.d +COPY supervisor/ckan_*.conf /etc/supervisord.d +COPY supervisor/supervisord.conf /etc/supervisord.conf ## Cron jobs config COPY setup/dbca_ckan_cron_jobs $APP_DIR diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index 3b9523bb..8b947768 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -38,7 +38,7 @@ export CKAN_INI=$APP_DIR/config/dbca.ini if [ $? -eq 0 ] then # Start supervisord - supervisord --configuration /etc/supervisord.conf & + su ckan -c "supervisord --configuration /etc/supervisord.conf &" # Start uwsgi su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' else diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index d5fdafeb..0d57cb67 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -96,7 +96,7 @@ if [ ! -f /tmp/container_ready ]; then fi # Start supervisord -supervisord --configuration /etc/supervisord.conf & +su ckan -c "supervisord --configuration /etc/supervisord.conf &" # Start the development server as the ckan user with automatic reload if [ "$USE_HTTPS_FOR_DEV" = true ] ; then diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index bc4cbf3b..898d0174 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -14,7 +14,7 @@ command= /usr/sbin/crond -f ; User the worker runs as. -user=root +user=ckan ; Start just a single worker. Increase this number if you have many or diff --git a/ckan/supervisor/supervisord.conf b/ckan/supervisor/supervisord.conf new file mode 100644 index 00000000..ef5ec08f --- /dev/null +++ b/ckan/supervisor/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = ckan:ckan + +[supervisord] +logfile = /srv/app/logs/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf From b4e8283e4d06be7534d224011678cf96f94512fc Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 1 May 2024 11:52:41 +1200 Subject: [PATCH 092/123] Refactor start scripts to use CKAN_INI variable for configuration file --- ckan/setup/dbca_start_ckan.sh.override | 6 ++++-- ckan/setup/dbca_start_ckan_development.sh.override | 9 ++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index 8b947768..ae78e9c7 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -2,12 +2,13 @@ # Set the CKAN plugins variable to the plugins value from the dbca.ini config file export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) + # Update the default ckan config file export CKAN_INI=$APP_DIR/ckan.ini # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -qE "beaker.session.secret ?= ?$" ckan.ini +if grep -qE "beaker.session.secret ?= ?$" $CKAN_INI then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" @@ -29,6 +30,7 @@ then *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; *) echo "$0: Ignoring $f (not an sh or py file)" ;; esac + echo done fi @@ -43,4 +45,4 @@ then su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' else echo "[prerun] failed...not starting CKAN." -fi \ No newline at end of file +fi diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index 0d57cb67..a16ac9c6 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Set the CKAN plugins variable to the plugins value from the dbca.ini config file export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) @@ -53,7 +53,7 @@ if [ ! -f /tmp/container_ready ]; then # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var - if grep -E "beaker.session.secret ?= ?$" ckan.ini + if grep -qE "beaker.session.secret ?= ?$" $CKAN_INI then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" @@ -88,9 +88,9 @@ if [ ! -f /tmp/container_ready ]; then done fi - # Revert back touse DBCA config file + # Revert back to use DBCA config file export CKAN_INI=$APP_DIR/config/dbca.ini - + # Set the container as ready so the startup scripts are not run again touch /tmp/container_ready fi @@ -104,4 +104,3 @@ if [ "$USE_HTTPS_FOR_DEV" = true ] ; then else su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" fi - From 87adac31e3e6cfb4499f8e9bf5b37b0978105e48 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 1 May 2024 13:29:30 +1200 Subject: [PATCH 093/123] Crond needs to be run from root user Output cron job logs to file --- ckan/setup/dbca_ckan_cron_jobs | 8 ++-- ckan/setup/dbca_start_ckan.sh.override | 7 ++-- .../dbca_start_ckan_development.sh.override | 8 ++-- ckan/supervisor/ckan_cron_jobs.conf | 42 ------------------- 4 files changed, 13 insertions(+), 52 deletions(-) delete mode 100644 ckan/supervisor/ckan_cron_jobs.conf diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index a3a4f064..ec95e75c 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,11 +1,11 @@ # Crontab for CKAN cron jobs # Hourly Rebuild Solr Index -0 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini ckan search-index +0 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini search-index rebuild >> /srv/app/logs/ckan-cron-jobs.log 2>&1 # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # 8am task to schedule embargo datasets to public visibility -0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets +0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets >> /srv/app/logs/ckan-cron-jobs.log 2>&1 # Midnight report generation for archiver broken links -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate >> /srv/app/logs/ckan-cron-jobs.log 2>&1 # 8am task to send CKAN email notifications -0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails \ No newline at end of file +0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails >> /srv/app/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index ae78e9c7..7e751622 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -1,8 +1,5 @@ #!/bin/bash -# Set the CKAN plugins variable to the plugins value from the dbca.ini config file -export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) - # Update the default ckan config file export CKAN_INI=$APP_DIR/ckan.ini @@ -36,11 +33,15 @@ fi # Revert back to use DBCA config file export CKAN_INI=$APP_DIR/config/dbca.ini +# Set the CKAN plugins variable to the plugins value from the dbca.ini config file +export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) if [ $? -eq 0 ] then # Start supervisord su ckan -c "supervisord --configuration /etc/supervisord.conf &" + # Start crond + /usr/sbin/crond # Start uwsgi su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' else diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index a16ac9c6..ce4839f6 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -1,8 +1,5 @@ #!/bin/bash -# Set the CKAN plugins variable to the plugins value from the dbca.ini config file -export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) - # Only run these start up scripts the first time the container is created if [ ! -f /tmp/container_ready ]; then # Install any local extensions in the src_extensions volume @@ -95,8 +92,13 @@ if [ ! -f /tmp/container_ready ]; then touch /tmp/container_ready fi +# Set the CKAN plugins variable to the plugins value from the dbca.ini config file +export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) + # Start supervisord su ckan -c "supervisord --configuration /etc/supervisord.conf &" +# Start crond +/usr/sbin/crond # Start the development server as the ckan user with automatic reload if [ "$USE_HTTPS_FOR_DEV" = true ] ; then diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf deleted file mode 100644 index 898d0174..00000000 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ /dev/null @@ -1,42 +0,0 @@ -; ======================================================= -; Supervisor configuration for CKAN cron jobs -; ======================================================= - -; 1. Copy this file to /etc/supervisor/conf.d -; 2. Make sure the paths below match your setup - - -[program:ckan-cron-jobs] - -; Use the full paths to the virtualenv and your configuration file here. -# This configuration sets up a cron job for CKAN. -command= /usr/sbin/crond -f - - -; User the worker runs as. -user=ckan - - -; Start just a single worker. Increase this number if you have many or -; particularly long running background jobs. -numprocs=1 -process_name=%(program_name)s-%(process_num)02d - - -; Log files. -stderr_logfile=/srv/app/logs/ckan-worker.log - - -; Make sure that the worker is started on system start and automatically -; restarted if it crashes unexpectedly. -autostart=true -autorestart=true - - -; Number of seconds the process has to run before it is considered to have -; started successfully. -startsecs=10 - -; Need to wait for currently executing tasks to finish at shutdown. -; Increase this if you have very long running tasks. -stopwaitsecs = 600 From 0f38743d54bad8cbdb2644a85dfa9acb29be54f3 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 1 May 2024 13:29:45 +1200 Subject: [PATCH 094/123] Refactor start scripts to improve initialisation process --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 101 ++++++++++++---------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index f3c32fd8..3cacccbd 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -1,50 +1,57 @@ #!/bin/bash -## Create logs folder/files and set permissions -mkdir -p $APP_DIR/logs -touch $APP_DIR/logs/ckan.log -touch $APP_DIR/logs/ckan-worker.log -chown -R ckan:ckan $APP_DIR/logs - -## Create archive folder and set permissions -mkdir -p $CKAN_STORAGE_PATH/archiver -chown -R ckan:ckan $CKAN_STORAGE_PATH/archiver - -echo "CKAN__PLUGINS: $CKAN__PLUGINS" - -if [[ $CKAN__PLUGINS == *"xloader"* ]]; then - CKAN_INI=$APP_DIR/ckan.ini - # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) - echo "Setting a temporary value for ckanext.xloader.api_token" - ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $APP_DIR/config/dbca.ini user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" +# Only run these start up scripts the first time the container is created +if [ ! -f /tmp/container_ready ]; then + ## Create logs folder/files and set permissions + mkdir -p $APP_DIR/logs + touch $APP_DIR/logs/ckan.log + touch $APP_DIR/logs/ckan-worker.log + chown -R ckan:ckan $APP_DIR/logs + + ## Create archive folder and set permissions + mkdir -p $CKAN_STORAGE_PATH/archiver + chown -R ckan:ckan $CKAN_STORAGE_PATH/archiver + + export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) + echo "CKAN__PLUGINS: $CKAN__PLUGINS" + + if [[ $CKAN__PLUGINS == *"xloader"* ]]; then + CKAN_INI=$APP_DIR/ckan.ini + # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) + echo "Setting a temporary value for ckanext.xloader.api_token" + ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $APP_DIR/config/dbca.ini user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" + fi + CKAN_INI=$APP_DIR/config/dbca.ini + + if [[ $CKAN__PLUGINS == *"archiver"* ]]; then + ckan -c $CKAN_INI archiver init + fi + + if [[ $CKAN__PLUGINS == *"report"* ]]; then + ckan -c $CKAN_INI report initdb + fi + + if [[ $CKAN__PLUGINS == *"qa"* ]]; then + ckan -c $CKAN_INI qa init + fi + + if [[ $CKAN__PLUGINS == *"pages"* ]]; then + ckan -c $CKAN_INI pages initdb + fi + + if [[ $CKAN__PLUGINS == *"doi"* ]]; then + ckan -c $CKAN_INI doi initdb + fi + + if [[ $CKAN__PLUGINS == *"dbca"* ]]; then + ckan -c $CKAN_INI db upgrade -p dbca + ckan -c $CKAN_INI dbca load_spatial_data + fi + + # if [[ $CKAN__PLUGINS == *"harvest"* ]]; then + # ckan -c $CKAN_INI db upgrade -p harvest + # fi + + # Set the container as ready so the startup scripts are not run again + touch /tmp/container_ready fi -CKAN_INI=$APP_DIR/config/dbca.ini - -if [[ $CKAN__PLUGINS == *"archiver"* ]]; then - ckan -c $CKAN_INI archiver init -fi - -if [[ $CKAN__PLUGINS == *"report"* ]]; then - ckan -c $CKAN_INI report initdb -fi - -if [[ $CKAN__PLUGINS == *"qa"* ]]; then - ckan -c $CKAN_INI qa init -fi - -if [[ $CKAN__PLUGINS == *"pages"* ]]; then - ckan -c $CKAN_INI pages initdb -fi - -if [[ $CKAN__PLUGINS == *"doi"* ]]; then - ckan -c $CKAN_INI doi initdb -fi - -if [[ $CKAN__PLUGINS == *"dbca"* ]]; then - ckan -c $CKAN_INI db upgrade -p dbca - ckan -c $CKAN_INI dbca load_spatial_data -fi - -# if [[ $CKAN__PLUGINS == *"harvest"* ]]; then -# ckan -c $CKAN_INI db upgrade -p harvest -# fi \ No newline at end of file From a9d886217bd8b6c21ab714225a0eeb13d4138d1f Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 1 May 2024 15:23:22 +1200 Subject: [PATCH 095/123] Removed nginx logrotate and cron jobs --- nginx/Dockerfile | 8 -------- nginx/setup/dbca_nginx_cron_jobs | 3 --- nginx/setup/dbca_nginx_rotate_logs | 9 --------- 3 files changed, 20 deletions(-) delete mode 100644 nginx/setup/dbca_nginx_cron_jobs delete mode 100644 nginx/setup/dbca_nginx_rotate_logs diff --git a/nginx/Dockerfile b/nginx/Dockerfile index aed247e3..8e6bbce5 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -13,13 +13,6 @@ COPY setup/default.conf.template ${NGINX_DIR}/templates/ RUN mkdir -p ${NGINX_DIR}/certs -# Install logrotate -RUN apk add logrotate -# Create logrotate config file for nginx logs -COPY setup/dbca_nginx_rotate_logs /etc/logrotate.d/dbca_nginx_rotate_logs -## Cron jobs config -COPY setup/dbca_nginx_cron_jobs /etc/crontabs/root - ENTRYPOINT \ openssl req \ -subj '/C=DE/ST=Berlin/L=Berlin/O=None/CN=localhost' \ @@ -28,5 +21,4 @@ ENTRYPOINT \ -keyout ${NGINX_DIR}/certs/ckan-local.key \ -out ${NGINX_DIR}/certs/ckan-local.crt \ -days 365 && \ - /usr/sbin/crond && \ /docker-entrypoint.sh nginx -g 'daemon off;' diff --git a/nginx/setup/dbca_nginx_cron_jobs b/nginx/setup/dbca_nginx_cron_jobs deleted file mode 100644 index 31921db7..00000000 --- a/nginx/setup/dbca_nginx_cron_jobs +++ /dev/null @@ -1,3 +0,0 @@ -# Crontab for Nginx cron jobs -# Midnight task to rotate CKAN app logs every day -0 0 * * * /usr/sbin/logrotate -s /srv/app/logs/logrotate.status /etc/logrotate.d/dbca_nginx_rotate_logs diff --git a/nginx/setup/dbca_nginx_rotate_logs b/nginx/setup/dbca_nginx_rotate_logs deleted file mode 100644 index 73145028..00000000 --- a/nginx/setup/dbca_nginx_rotate_logs +++ /dev/null @@ -1,9 +0,0 @@ -/srv/app/logs/nginx_access.log /srv/app/logs/nginx_error.log { - daily - rotate 7 - missingok - notifempty - compress - delaycompress - copytruncate -} \ No newline at end of file From cfb5d8c0bc9ff47db542a99715d144417367b5a3 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 2 May 2024 08:59:03 +0800 Subject: [PATCH 096/123] Add instructions for deleting and reloading spatial data in README.MD --- spatial_data/README.MD | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spatial_data/README.MD b/spatial_data/README.MD index 9f5aee6a..9a6cd314 100644 --- a/spatial_data/README.MD +++ b/spatial_data/README.MD @@ -70,3 +70,19 @@ Example: ``` ckan -c $CKAN_INI dbca load_spatial_data ``` + + ## Delete dataset spatial metadata values + + If there is a case where all or most current dataset spatial coordinates are incorrect and they need to be removed, run the below commands + + Set dataset spatial values you to NULL in the database + ``` + update "public"."package_extra" + set value = null + where key = 'spatial'; + ``` + + Rebuild Solr index by running the CLI command in either the ckan or ckan-worker containers + ``` + ckan -c $CKAN_INI search-index rebuild + ``` \ No newline at end of file From 152015d77a3764aee761b790cc883eb44bdd04de Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 2 May 2024 09:46:03 +0800 Subject: [PATCH 097/123] Update CKAN configuration to set temporary value for user $USER for ckanext.xloader.api_token --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 3cacccbd..e58b8e25 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -17,9 +17,12 @@ if [ ! -f /tmp/container_ready ]; then if [[ $CKAN__PLUGINS == *"xloader"* ]]; then CKAN_INI=$APP_DIR/ckan.ini + # Use the CKAN_SYSADMIN_NAME or CKAN_SITE_ID to create a token for the xloader user + export CKAN_SITE_ID=$(grep '^ckan\.site_id ' $CKAN_INI | cut -d'=' -f2) + USER=${CKAN_SYSADMIN_NAME:-$CKAN_SITE_ID} # Add ckan.xloader.api_token to the CKAN config file (updated with corrected value later) - echo "Setting a temporary value for ckanext.xloader.api_token" - ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $APP_DIR/config/dbca.ini user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" + echo "Setting a temporary value for user $USER for ckanext.xloader.api_token" + ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $APP_DIR/config/dbca.ini user token add $USER xloader | tail -n 1 | tr -d '\t')" fi CKAN_INI=$APP_DIR/config/dbca.ini From d31f9e9da842b239ce5283d0fecebd166ac2ec52 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 7 May 2024 11:54:56 +1200 Subject: [PATCH 098/123] Added logic to start background services only for the ckan worker container and the web server only for ckan container --- ckan/Dockerfile.worker | 1 + ckan/setup/dbca_start_ckan.sh.override | 17 +++++++++----- .../dbca_start_ckan_development.sh.override | 22 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 5e11554d..07083f64 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -1,6 +1,7 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} +ENV CKAN_WORKER=true # Override the default CKAN config plugins to only use the ones we need for the worker ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi datatables_view envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index 7e751622..4923aee8 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -38,12 +38,17 @@ export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'= if [ $? -eq 0 ] then - # Start supervisord - su ckan -c "supervisord --configuration /etc/supervisord.conf &" - # Start crond - /usr/sbin/crond - # Start uwsgi - su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' + # if CKAN_WORKER is set to true, start the background services + if [ "$CKAN_WORKER" = "true" ] + then + # Start supervisord + su ckan -c "supervisord --configuration /etc/supervisord.conf" + # Start crond + /usr/sbin/crond + else + # Start uwsgi + su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' + fi else echo "[prerun] failed...not starting CKAN." fi diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index ce4839f6..450ab551 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -95,14 +95,18 @@ fi # Set the CKAN plugins variable to the plugins value from the dbca.ini config file export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) -# Start supervisord -su ckan -c "supervisord --configuration /etc/supervisord.conf &" -# Start crond -/usr/sbin/crond - -# Start the development server as the ckan user with automatic reload -if [ "$USE_HTTPS_FOR_DEV" = true ] ; then - su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0 -C unsafe.cert -K unsafe.key" +# if CKAN_WORKER is set to true, start the background services +if [ "$CKAN_WORKER" = "true" ] +then + # Start crond + /usr/sbin/crond + # Start supervisord + su ckan -c "supervisord --configuration /etc/supervisord.conf" else - su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" + # Start the development server as the ckan user with automatic reload + if [ "$USE_HTTPS_FOR_DEV" = true ] ; then + su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0 -C unsafe.cert -K unsafe.key" + else + su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" + fi fi From 7f33ccea1cd48fd371f09ae07fb1ee6d96609a3f Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 7 May 2024 12:49:47 +1200 Subject: [PATCH 099/123] Updated ckan and ckan-worker healthcheck test commands --- docker-compose.dev.yml | 4 ++-- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index bbf4edb9..5a5ebba3 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -36,7 +36,7 @@ services: - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + test: ["CMD", "curl", "-I", "http://localhost:5000"] stdin_open: true tty: true @@ -68,7 +68,7 @@ services: - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + test: ["CMD-SHELL", "supervisorctl", "status"] stdin_open: true tty: true diff --git a/docker-compose.yml b/docker-compose.yml index ad5ffc5c..275511e6 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + test: ["CMD", "curl", "-I", "http://localhost:5000"] ckan-worker: container_name: ${WORKER_CONTAINER_NAME} @@ -94,7 +94,7 @@ services: - ./spatial_data:/srv/app/spatial_data restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] + test: ["CMD-SHELL", "supervisorctl", "status"] db: container_name: ${POSTGRESQL_CONTAINER_NAME} From 05cd77940a5397a2859a6b4169f9580e389d03d9 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 7 May 2024 12:10:41 +0800 Subject: [PATCH 100/123] Update devcontainer.json with new settings --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a780382..ccf95ef2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -59,7 +59,8 @@ "--max-line-length", "160" ], - "cSpell.language": "en-AU" + "cSpell.language": "en-AU", + "extensions.verifySignature": false } } }, From 4a218e4a978c8b700ecb6b84d65badd8b91b5fd0 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 7 May 2024 12:51:02 +0800 Subject: [PATCH 101/123] Update CKAN configuration to include webpage_view plugin and default view --- ckan/config/dbca.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 424f0609..7f058999 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view resource_proxy datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report envvars +ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view resource_proxy datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view +ckan.views.default_views = image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU From 52dbb9354a72ef64b49e3e3421c06eebb3211001 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 8 May 2024 12:53:47 +1200 Subject: [PATCH 102/123] Removed filehandler for logs --- ckan/Dockerfile.worker | 2 -- ckan/config/dbca.ini | 18 ++++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 07083f64..3ba3cb85 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -5,8 +5,6 @@ ENV CKAN_WORKER=true # Override the default CKAN config plugins to only use the ones we need for the worker ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi datatables_view envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" -# Update the CKAN config log file handler to use the worker log file -RUN ckan config-tool $CKAN_INI -s handler_fileHandler "args = ('/srv/app/logs/ckan-worker.log', 'midnight', 1, 7)" ## Supervisor config COPY supervisor/ckan_*.conf /etc/supervisord.d diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 7f058999..3f78eac2 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -130,36 +130,36 @@ ckanext.saml2auth.logout_expected_binding = urn:oasis:names:tc:SAML:2.0:bindings keys = root, ckan, ckanext, werkzeug, saml2 [handlers] -keys = console, fileHandler +keys = console [formatters] keys = generic [logger_root] level = WARNING -handlers = console, fileHandler +handlers = console [logger_werkzeug] level = WARNING -handlers = console, fileHandler +handlers = console qualname = werkzeug propagate = 0 [logger_ckan] level = INFO -handlers = console, fileHandler +handlers = console qualname = ckan propagate = 0 [logger_ckanext] level = DEBUG -handlers = console, fileHandler +handlers = console qualname = ckanext propagate = 0 [logger_saml2] level = DEBUG -handlers = console, fileHandler +handlers = console qualname = saml2 propagate = 0 @@ -169,11 +169,5 @@ args = (sys.stderr,) level = NOTSET formatter = generic -[handler_fileHandler] -class = handlers.TimedRotatingFileHandler -args = ('/srv/app/logs/ckan.log', 'midnight', 1, 7) -level = NOTSET -formatter = generic - [formatter_generic] format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s From 43cf3e8b36606dbcdcbdfc5dcfd8238530853fb5 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 8 May 2024 13:47:49 +1200 Subject: [PATCH 103/123] Create webassets folder and set permissions --- ckan/config/dbca.ini | 2 ++ ckan/docker-entrypoint.d/02_setup_dbca.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index 3f78eac2..b63b3ea5 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -33,6 +33,8 @@ ckan.activity_streams_email_notifications = true # ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 ckan.search.solr_allowed_query_parsers = frange +## Webassets Settings ########################################################## +ckan.webassets.path = /srv/app/webassets ## CKAN Extensions configuration ############################################### diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index e58b8e25..17143b5f 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -2,19 +2,23 @@ # Only run these start up scripts the first time the container is created if [ ! -f /tmp/container_ready ]; then + export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) + echo "CKAN__PLUGINS: $CKAN__PLUGINS" + ## Create logs folder/files and set permissions mkdir -p $APP_DIR/logs - touch $APP_DIR/logs/ckan.log touch $APP_DIR/logs/ckan-worker.log chown -R ckan:ckan $APP_DIR/logs + ## Create webassets folder and set permissions + mkdir -p $APP_DIR/webassets + chown -R ckan:ckan $APP_DIR/webassets + ckan -c $CKAN_INI asset build + ## Create archive folder and set permissions mkdir -p $CKAN_STORAGE_PATH/archiver chown -R ckan:ckan $CKAN_STORAGE_PATH/archiver - export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) - echo "CKAN__PLUGINS: $CKAN__PLUGINS" - if [[ $CKAN__PLUGINS == *"xloader"* ]]; then CKAN_INI=$APP_DIR/ckan.ini # Use the CKAN_SYSADMIN_NAME or CKAN_SITE_ID to create a token for the xloader user From c0d2c17d295e83f065e23d653fb438fee3c77493 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 9 May 2024 11:49:46 +1200 Subject: [PATCH 104/123] Add view plugins --- ckan/Dockerfile.worker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 3ba3cb85..59265140 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -3,7 +3,7 @@ FROM ${CKAN_IMAGE} ENV CKAN_WORKER=true # Override the default CKAN config plugins to only use the ones we need for the worker -ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi datatables_view envvars" +ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" ## Supervisor config From 4a976179836552106109a586c419322d3a087f73 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 16 May 2024 09:13:50 +0800 Subject: [PATCH 105/123] Updated patch --- .../01_copy_archive_resource.patch | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch index ad48a5c6..11955c3f 100644 --- a/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch +++ b/ckan/patches/ckanext-archiver/01_copy_archive_resource.patch @@ -1,6 +1,16 @@ -index 06a1758..eac41f3 100644 +diff --git a/ckanext/archiver/tasks.py b/ckanext/archiver/tasks.py +index 06a1758..cbfbdaf 100644 --- a/ckanext/archiver/tasks.py +++ b/ckanext/archiver/tasks.py +@@ -275,7 +275,7 @@ def _update_resource(resource_id, queue, log): + return + + mimetype = None +- headers = None ++ headers = {} + content_type, content_encoding = mimetypes.guess_type(url) + if content_type: + mimetype = _clean_content_type(content_type) @@ -568,14 +568,13 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): except Exception: file_name = "resource" @@ -21,7 +31,3 @@ index 06a1758..eac41f3 100644 raise log.info('Archived resource as: %s', saved_file) - -~ -~ -~ \ No newline at end of file From 01c7a8dacc24a5e40479ce7ebcb0cfda405776b3 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 17 May 2024 09:57:51 +1200 Subject: [PATCH 106/123] chore: Download additional CKAN logs from Azure File Share --- .ahoy.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.ahoy.yml b/.ahoy.yml index 98b1c7b3..80501fef 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -213,6 +213,11 @@ commands: echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + + LOG_FILE="ckan-cron-jobs.log" + echo "Downloading $LOG_FILE" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" LOG_FILE="nginx_access.log" echo "Downloading $LOG_FILE" @@ -223,5 +228,10 @@ commands: echo "Downloading $LOG_FILE" FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" + + LOG_FILE="supervisord.log" + echo "Downloading $LOG_FILE" + FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN" + curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL" echo "CKAN logs downloaded to dbca/ckan_logs" \ No newline at end of file From 42c0b782baefd08798bbe56dc5a7093bab275198 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Mon, 20 May 2024 08:50:55 +0800 Subject: [PATCH 107/123] Add manual trigger for workflow_dispatch in dbca_build.yml --- .github/workflows/dbca_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dbca_build.yml b/.github/workflows/dbca_build.yml index 72459033..ed817601 100644 --- a/.github/workflows/dbca_build.yml +++ b/.github/workflows/dbca_build.yml @@ -7,6 +7,8 @@ on: - master - develop - tags/* + # Trigger the workflow manually + workflow_dispatch: env: REGISTRY: ghcr.io From 5b299d2cbfbfc6fae2de821971806b623c4ce7d7 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Mon, 20 May 2024 09:55:48 +0800 Subject: [PATCH 108/123] Disable DCAT translation keys https://github.com/ckan/ckanext-dcat/issues/185 --- ckan/config/dbca.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index b63b3ea5..e7d4bfc1 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -126,6 +126,9 @@ ckanext.saml2auth.logout_requests_signed = False # Default: urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST ckanext.saml2auth.logout_expected_binding = urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect +# ckanext-dcat +ckanext.dcat.translate_keys = False + ## Logging configuration [loggers] From ebc0d816abc1b5399af71818c547c257586ec44d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 28 May 2024 10:36:31 +1200 Subject: [PATCH 109/123] Update cron job schedule for Solr index rebuild and archiver report generation --- ckan/setup/dbca_ckan_cron_jobs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index ec95e75c..d18efa58 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,11 +1,11 @@ # Crontab for CKAN cron jobs -# Hourly Rebuild Solr Index -0 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini search-index rebuild >> /srv/app/logs/ckan-cron-jobs.log 2>&1 +# Midnight Rebuild Solr Index +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini search-index rebuild >> /srv/app/logs/ckan-cron-jobs.log 2>&1 # Example cron job runs the harvester run command every 15 mins #*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # 8am task to schedule embargo datasets to public visibility 0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets >> /srv/app/logs/ckan-cron-jobs.log 2>&1 -# Midnight report generation for archiver broken links -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate >> /srv/app/logs/ckan-cron-jobs.log 2>&1 +# 12:30am report generation for archiver broken links +30 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate >> /srv/app/logs/ckan-cron-jobs.log 2>&1 # 8am task to send CKAN email notifications 0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails >> /srv/app/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file From 5fdfe7be8ead897da721671db05f1909a910dc7d Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 28 May 2024 13:34:08 +1200 Subject: [PATCH 110/123] Start supervisord before crond in dbca_start_ckan.sh.override --- ckan/setup/dbca_start_ckan.sh.override | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override index 4923aee8..48b94307 100755 --- a/ckan/setup/dbca_start_ckan.sh.override +++ b/ckan/setup/dbca_start_ckan.sh.override @@ -41,10 +41,10 @@ then # if CKAN_WORKER is set to true, start the background services if [ "$CKAN_WORKER" = "true" ] then - # Start supervisord - su ckan -c "supervisord --configuration /etc/supervisord.conf" # Start crond /usr/sbin/crond + # Start supervisord + su ckan -c "supervisord --configuration /etc/supervisord.conf" else # Start uwsgi su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini' From 50cc34ce0c4fd3e303b8be67f4ea29d4f5fcdab1 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 28 May 2024 10:56:43 +0800 Subject: [PATCH 111/123] Update CKAN extensions and requirements --- ckan/setup/dbca_requirements.sh | 14 +++++--------- src/dbca_install_extensions.sh | 4 ---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index 91bd0b64..63124b34 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -29,8 +29,8 @@ pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext pip3 install -r ${SRC_DIR}/ckanext-report/requirements.txt # # Showcase -# pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.6.1#egg=ckanext-showcase -# pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt +pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.7.0egg=ckanext-showcase +pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt # Scheming pip3 install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming @@ -58,6 +58,9 @@ pip3 install -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckane # DOI # pip3 install -e git+https://github.com/NaturalHistoryMuseum/ckanext-doi@v3.1.10#egg=ckanext-doi +# Office Docs +pip3 install -e git+https://github.com/jqnatividad/ckanext-officedocs.git@master#egg=ckanext-officedocs + # SAML2 pip3 install -e git+https://github.com/keitaroinc/ckanext-saml2auth.git@v1.3.0#egg=ckanext-saml2auth @@ -77,10 +80,3 @@ rm /tmp/qsv.zip apk add file pip3 install -e git+https://github.com/dbca-wa/ckanext-qa.git@develop#egg=ckanext-qa pip3 install -r ${SRC_DIR}/ckanext-qa/requirements.txt - -# Office Docs -pip3 install -e git+https://github.com/dbca-wa/ckanext-officedocs.git@develop#egg=ckanext-officedocs - -# Showcase -pip3 install -e git+https://github.com/dbca-wa/ckanext-showcase.git@develop#egg=ckanext-showcase -pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt diff --git a/src/dbca_install_extensions.sh b/src/dbca_install_extensions.sh index a279bcf2..b14ca3ef 100644 --- a/src/dbca_install_extensions.sh +++ b/src/dbca_install_extensions.sh @@ -11,10 +11,6 @@ git clone https://github.com/dbca-wa/ckanext-dbca.git git clone https://github.com/dbca-wa/ckanext-qa.git #These extensions will be installed by default, but we don't want them sed -i".$(date +%Y%m%d_%H%M%S).bak" -e '/ckanext-report/d' -e '/ckanext-archiver/d' ckanext-qa/dev-requirements.txt -# Office Docs -git clone https://github.com/dbca-wa/ckanext-officedocs.git -# Showcase -git clone https://github.com/dbca-wa/ckanext-showcase.git # DOI git clone https://github.com/dbca-wa/ckanext-doi From db2e51d94ff064e8fede9c805dc16ae0ab4f3bb2 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 4 Jun 2024 11:40:40 +1200 Subject: [PATCH 112/123] Fixed showcase install missing hash character Updated report to ignore existing install from archiver requirements. --- ckan/setup/dbca_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index 63124b34..be9a1f64 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -25,11 +25,11 @@ pip3 install -e git+https://github.com/ckan/ckanext-pages.git@v0.5.2#egg=ckanext pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview # Report -pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext-report +pip3 install -e git+http://github.com/ckan/ckanext-report.git@master#egg=ckanext-report --exists-action i pip3 install -r ${SRC_DIR}/ckanext-report/requirements.txt # # Showcase -pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.7.0egg=ckanext-showcase +pip3 install -e git+https://github.com/ckan/ckanext-showcase.git@v1.7.0#egg=ckanext-showcase pip3 install -r ${SRC_DIR}/ckanext-showcase/requirements.txt # Scheming From b711d03118e409cf49d5780a49a0016c490c3fec Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 4 Jun 2024 15:24:36 +1200 Subject: [PATCH 113/123] Set CKAN_INI to dbca.ini for asset build --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 17143b5f..29360c58 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -2,6 +2,7 @@ # Only run these start up scripts the first time the container is created if [ ! -f /tmp/container_ready ]; then + CKAN_INI=$APP_DIR/config/dbca.ini export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2) echo "CKAN__PLUGINS: $CKAN__PLUGINS" From d3155f0313c780e69ecbbfc9187fa0447fa9638a Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Wed, 5 Jun 2024 14:20:22 +1200 Subject: [PATCH 114/123] Update CKAN asset build command to run as the 'ckan' user --- ckan/docker-entrypoint.d/02_setup_dbca.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh index 29360c58..49e86497 100644 --- a/ckan/docker-entrypoint.d/02_setup_dbca.sh +++ b/ckan/docker-entrypoint.d/02_setup_dbca.sh @@ -14,7 +14,7 @@ if [ ! -f /tmp/container_ready ]; then ## Create webassets folder and set permissions mkdir -p $APP_DIR/webassets chown -R ckan:ckan $APP_DIR/webassets - ckan -c $CKAN_INI asset build + su ckan -c "ckan -c $CKAN_INI asset build" ## Create archive folder and set permissions mkdir -p $CKAN_STORAGE_PATH/archiver From c5f86ad60ea3129c9faeb303e78c2743bce5f2f7 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 6 Jun 2024 11:47:41 +0800 Subject: [PATCH 115/123] chore: Fix uploader to handle I/O error when removing temporary file --- .../ckan/01_uploader_remove_temp_file.patch | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ckan/patches/ckan/01_uploader_remove_temp_file.patch diff --git a/ckan/patches/ckan/01_uploader_remove_temp_file.patch b/ckan/patches/ckan/01_uploader_remove_temp_file.patch new file mode 100644 index 00000000..3aa934b9 --- /dev/null +++ b/ckan/patches/ckan/01_uploader_remove_temp_file.patch @@ -0,0 +1,16 @@ +diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py +index 82e2dcbaf..56a9db59d 100644 +--- a/ckan/lib/uploader.py ++++ b/ckan/lib/uploader.py +@@ -345,8 +345,9 @@ class ResourceUpload(object): + except logic.ValidationError: + os.remove(tmp_filepath) + raise +- finally: +- self.upload_file.close() ++ except OSError as e: ++ # Handle I/O error when removing the temporary file ++ log.warning(f"Failed to remove temporary file: {tmp_filepath}. Error: {e}") + os.rename(tmp_filepath, filepath) + return + From 685c2b3a3e1f428612dcc8c330d5bffdc3fc20d4 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 7 Jun 2024 08:26:06 +0800 Subject: [PATCH 116/123] chore: Improve uploader to handle I/O error when removing temporary file --- .../ckan/01_uploader_remove_temp_file.patch | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/ckan/patches/ckan/01_uploader_remove_temp_file.patch b/ckan/patches/ckan/01_uploader_remove_temp_file.patch index 3aa934b9..4378dab5 100644 --- a/ckan/patches/ckan/01_uploader_remove_temp_file.patch +++ b/ckan/patches/ckan/01_uploader_remove_temp_file.patch @@ -1,16 +1,28 @@ diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py -index 82e2dcbaf..56a9db59d 100644 +index 82e2dcbaf..e44fc5f65 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py -@@ -345,8 +345,9 @@ class ResourceUpload(object): +@@ -343,7 +343,22 @@ class ResourceUpload(object): + try: + _copy_file(self.upload_file, output_file, max_size) except logic.ValidationError: - os.remove(tmp_filepath) +- os.remove(tmp_filepath) ++ try: ++ os.remove(tmp_filepath) ++ except OSError: ++ # Other process (blobfuse) might be still using file ++ # Sleep for a second and try again 5 times ++ import time ++ max_attempts = 5 ++ delay = .5 # delay in seconds ++ for attempt in range(max_attempts): ++ time.sleep(delay) ++ try: ++ os.remove(tmp_filepath) ++ except OSError as e: ++ log.warning(f"Failed to remove temporary file attempt {attempt + 1}: {tmp_filepath}. Error: {e}") ++ except Exception as e: ++ log.error(f"Failed to remove temporary file: {tmp_filepath}. Error: {e}") raise -- finally: -- self.upload_file.close() -+ except OSError as e: -+ # Handle I/O error when removing the temporary file -+ log.warning(f"Failed to remove temporary file: {tmp_filepath}. Error: {e}") - os.rename(tmp_filepath, filepath) - return - + finally: + self.upload_file.close() From 68bf9c23dc394da999201b0005893db55942ce2a Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 7 Jun 2024 12:35:23 +0800 Subject: [PATCH 117/123] chore: Improve uploader to handle I/O error when removing temporary file --- .../ckan/01_uploader_remove_temp_file.patch | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ckan/patches/ckan/01_uploader_remove_temp_file.patch b/ckan/patches/ckan/01_uploader_remove_temp_file.patch index 4378dab5..8de2ee1b 100644 --- a/ckan/patches/ckan/01_uploader_remove_temp_file.patch +++ b/ckan/patches/ckan/01_uploader_remove_temp_file.patch @@ -1,28 +1,28 @@ diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py -index 82e2dcbaf..e44fc5f65 100644 +index 82e2dcbaf..005645485 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py -@@ -343,7 +343,22 @@ class ResourceUpload(object): - try: +@@ -338,15 +338,15 @@ class ResourceUpload(object): + if e.errno != 17: + raise + tmp_filepath = filepath + '~' +- with open(tmp_filepath, 'wb+') as output_file: +- assert self.upload_file +- try: ++ try: ++ with open(tmp_filepath, 'wb+') as output_file: ++ assert self.upload_file _copy_file(self.upload_file, output_file, max_size) - except logic.ValidationError: +- except logic.ValidationError: - os.remove(tmp_filepath) -+ try: -+ os.remove(tmp_filepath) -+ except OSError: -+ # Other process (blobfuse) might be still using file -+ # Sleep for a second and try again 5 times -+ import time -+ max_attempts = 5 -+ delay = .5 # delay in seconds -+ for attempt in range(max_attempts): -+ time.sleep(delay) -+ try: -+ os.remove(tmp_filepath) -+ except OSError as e: -+ log.warning(f"Failed to remove temporary file attempt {attempt + 1}: {tmp_filepath}. Error: {e}") -+ except Exception as e: -+ log.error(f"Failed to remove temporary file: {tmp_filepath}. Error: {e}") - raise - finally: - self.upload_file.close() +- raise +- finally: +- self.upload_file.close() ++ except logic.ValidationError: ++ os.remove(tmp_filepath) ++ raise ++ finally: ++ self.upload_file.close() + os.rename(tmp_filepath, filepath) + return + From 326f6a05fbfb5d88cc1ea09c23283019019e8c31 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 25 Jun 2024 11:48:45 +0800 Subject: [PATCH 118/123] Disabled webpage_view plugin and view --- ckan/Dockerfile.worker | 2 +- ckan/config/dbca.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 59265140..d170e6f6 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -3,7 +3,7 @@ FROM ${CKAN_IMAGE} ENV CKAN_WORKER=true # Override the default CKAN config plugins to only use the ones we need for the worker -ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view envvars" +ENV CKAN__PLUGINS="dbca activity datastore xloader scheming_datasets qa archiver report doi image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view envvars" RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}" ## Supervisor config diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index e7d4bfc1..91aff776 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -10,10 +10,10 @@ use = config:/srv/app/ckan.ini ckan.devserver.watch_patterns = /srv/app/ckan.ini ## Plugins Settings ############################################################ -ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view resource_proxy datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report envvars +ckan.plugins = dbca saml2auth activity image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view resource_proxy datastore xloader pages showcase hierarchy_display hierarchy_form hierarchy_group_form dcat scheming_datasets spatial_metadata spatial_query doi qa archiver report envvars ## Resource Views Settings ##################################################### -ckan.views.default_views = image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view webpage_view +ckan.views.default_views = image_view text_view datatables_view pdf_view geo_view shp_view officedocs_view ## Internationalisation Settings ############################################### ckan.locale_default = en_AU From 1324a7e84e30e3e740db40f67da5d946543f4077 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 2 Jul 2024 07:19:11 +0800 Subject: [PATCH 119/123] Update db-import command to remove /tmp/container_ready so the start up scripts are run --- .ahoy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ahoy.yml b/.ahoy.yml index 80501fef..3c0fb7b3 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -133,7 +133,7 @@ commands: # Import the database docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@" # Initialise plugin tables - docker compose -f $DOCKER_COMPOSE exec -T $CKAN_CONTAINER_NAME sh -c 'export CKAN__PLUGINS=$(grep "^ckan\.plugins" $APP_DIR/config/dbca.ini | cut -d"=" -f2) && bash /docker-entrypoint.d/02_setup_dbca.sh' + docker compose -f $DOCKER_COMPOSE exec -T $CKAN_CONTAINER_NAME sh -c 'rm /tmp/container_ready && export CKAN__PLUGINS=$(grep "^ckan\.plugins" $APP_DIR/config/dbca.ini | cut -d"=" -f2) && bash /docker-entrypoint.d/02_setup_dbca.sh' # Rebuild the Solr index ahoy reindex-solr else From c0567b0a08d284c40a9a5a2277e46e71fca5ae94 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 2 Jul 2024 11:21:23 +1200 Subject: [PATCH 120/123] Update pip install commands to handle existing packages --- ckan/setup/dbca_start_ckan_development.sh.override | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override index 450ab551..245a8a38 100644 --- a/ckan/setup/dbca_start_ckan_development.sh.override +++ b/ckan/setup/dbca_start_ckan_development.sh.override @@ -13,17 +13,17 @@ if [ ! -f /tmp/container_ready ]; then if [ -f $i/pip-requirements.txt ]; then - pip install -r $i/pip-requirements.txt + pip install -r $i/pip-requirements.txt --exists-action i echo "Found requirements file in $i" fi if [ -f $i/requirements.txt ]; then - pip install -r $i/requirements.txt + pip install -r $i/requirements.txt --exists-action i echo "Found requirements file in $i" fi if [ -f $i/dev-requirements.txt ]; then - pip install -r $i/dev-requirements.txt + pip install -r $i/dev-requirements.txt --exists-action i echo "Found dev-requirements file in $i" fi if [ -f $i/setup.py ]; From 5af8a56d2f9c8926d6779220d91f4321dad691e7 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 2 Jul 2024 13:42:56 +1200 Subject: [PATCH 121/123] Fix numpy version incompatibility --- ckan/setup/dbca_requirements.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh index be9a1f64..f408df5d 100644 --- a/ckan/setup/dbca_requirements.sh +++ b/ckan/setup/dbca_requirements.sh @@ -44,6 +44,8 @@ apk add --no-cache \ proj-util \ proj-dev pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial +# Incompatibility with Shapely >2 Due to NumPy 2.0 Update https://github.com/ckan/ckanext-spatial/issues/330 +pip3 install "numpy>=1.26,<2" pip3 install -r ${SRC_DIR}/ckanext-spatial/requirements.txt # XLoader From 9bc8e3f57ce61d94f9cb3651c2e37d11cf9e63f7 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Tue, 2 Jul 2024 13:44:04 +1200 Subject: [PATCH 122/123] Update .gitignore to include logs directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7f3e9f9e..eed334b9 100755 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ local/* # DBCA src/ dbca/ -spatial_data/*.geojson \ No newline at end of file +spatial_data/*.geojson +logs/ \ No newline at end of file From f4bbd7a9401fe88f3d32ffa03f17d321a96be980 Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Fri, 5 Jul 2024 09:35:53 +0800 Subject: [PATCH 123/123] Update SAML2AuthPlugin to logout user on session expiration --- .../ckanext-saml2auth/01_logout_user.patch | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 ckan/patches/ckanext-saml2auth/01_logout_user.patch diff --git a/ckan/patches/ckanext-saml2auth/01_logout_user.patch b/ckan/patches/ckanext-saml2auth/01_logout_user.patch new file mode 100644 index 00000000..14d17872 --- /dev/null +++ b/ckan/patches/ckanext-saml2auth/01_logout_user.patch @@ -0,0 +1,22 @@ +diff --git a/ckanext/saml2auth/plugin.py b/ckanext/saml2auth/plugin.py +index 53b8454..07a3307 100644 +--- a/ckanext/saml2auth/plugin.py ++++ b/ckanext/saml2auth/plugin.py +@@ -22,6 +22,7 @@ from saml2.client_base import LogoutError + from saml2 import entity + + from flask import session, redirect, make_response ++from flask_login import logout_user + + import ckan.plugins as plugins + import ckan.plugins.toolkit as toolkit +@@ -103,7 +104,8 @@ class Saml2AuthPlugin(plugins.SingletonPlugin): + domain = h.get_site_domain_for_cookie() + # Clear session cookie in the browser + response.set_cookie('ckan', domain=domain, expires=0) +- ++ # logout user from CKAN ++ logout_user() + if not toolkit.check_ckan_version(min_version="2.10"): + # CKAN <= 2.9.x also sets auth_tkt cookie + response.set_cookie('auth_tkt', domain=domain, expires=0) \ No newline at end of file