From e232ce297de1cc49177ee1bb3d4b542cb484fb7f Mon Sep 17 00:00:00 2001 From: Mark Calvert Date: Thu, 1 Feb 2024 11:40:23 +1300 Subject: [PATCH] Updated project to use log files for ckan, ckan-worker, ckan-cron-jobs, nginx --- ckan/Dockerfile | 6 ++++++ ckan/Dockerfile.dev | 6 ++++++ ckan/Dockerfile.worker | 6 ------ ckan/config/dbca.ini | 16 +++++++++++----- ckan/setup/dbca_ckan_cron_jobs | 6 +++--- ckan/supervisor/ckan_cron_jobs.conf | 8 +++++--- ckan/supervisor/ckan_worker_bulk.conf | 6 ++++-- ckan/supervisor/ckan_worker_default.conf | 6 ++++-- ckan/supervisor/ckan_worker_priority.conf | 6 ++++-- docker-compose.dev.yml | 4 ++++ docker-compose.yml | 4 ++++ nginx/setup/default.conf | 5 ++++- nginx/setup/default.conf.template | 5 ++++- 13 files changed, 59 insertions(+), 25 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 1b0176da..46cf8c80 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -24,6 +24,12 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +## Create logs folder +RUN mkdir -p $APP_DIR/logs && \ + touch $APP_DIR/logs/ckan.log && \ + chown -R ckan:ckan $APP_DIR/logs && \ + chmod -R 755 $APP_DIR/logs + # Override the default start_ckan.sh script COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh RUN chmod +x ${APP_DIR}/start_ckan.sh diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 0fb294fa..aa59b66c 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,6 +57,12 @@ RUN for d in $APP_DIR/patches/*; do \ ## DBCA specific configuration ## +## Create logs folder +RUN mkdir -p $APP_DIR/logs && \ + touch $APP_DIR/logs/ckan.log && \ + chown -R ckan:ckan $APP_DIR/logs && \ + chmod -R 755 $APP_DIR/logs + # Override the default dbca_start_ckan_development.sh script COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh RUN chmod +x ${APP_DIR}/start_ckan_development.sh diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker index 4c8a4657..03853739 100644 --- a/ckan/Dockerfile.worker +++ b/ckan/Dockerfile.worker @@ -1,12 +1,6 @@ ARG CKAN_IMAGE FROM ${CKAN_IMAGE} -## Create logs folder -RUN mkdir -p $APP_DIR/logs && \ - touch $APP_DIR/logs/ckan-cron-jobs.log && \ - chown -R ckan:ckan $APP_DIR/logs && \ - chmod -R 755 $APP_DIR/logs - ## Supervisor config COPY supervisor/*.conf /etc/supervisord.d diff --git a/ckan/config/dbca.ini b/ckan/config/dbca.ini index b6e6174f..2f423bb1 100644 --- a/ckan/config/dbca.ini +++ b/ckan/config/dbca.ini @@ -57,30 +57,30 @@ ckanext.qa.qsv_bin=/usr/local/bin/qsv keys = root, ckan, ckanext, werkzeug [handlers] -keys = console +keys = console, fileHandler [formatters] keys = generic [logger_root] level = WARNING -handlers = console +handlers = console, fileHandler [logger_werkzeug] level = WARNING -handlers = console +handlers = console, fileHandler qualname = werkzeug propagate = 0 [logger_ckan] level = INFO -handlers = console +handlers = console, fileHandler qualname = ckan propagate = 0 [logger_ckanext] level = DEBUG -handlers = console +handlers = console, fileHandler qualname = ckanext propagate = 0 @@ -90,5 +90,11 @@ args = (sys.stderr,) level = NOTSET formatter = generic +[handler_fileHandler] +class = handlers.TimedRotatingFileHandler +args = ('/srv/app/logs/ckan.log', 'midnight', 1, 0) +level = NOTSET +formatter = generic + [formatter_generic] format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs index 5c9442ff..618ccd81 100644 --- a/ckan/setup/dbca_ckan_cron_jobs +++ b/ckan/setup/dbca_ckan_cron_jobs @@ -1,7 +1,7 @@ # Crontab for CKAN cron jobs # Example cron job runs the harvester run command every 15 mins -#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run # Midnight task to schedule embargo datasets to public visibility -0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets # Midnight report generation for archiver broken links -0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini report generate >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1 \ No newline at end of file +0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate \ No newline at end of file diff --git a/ckan/supervisor/ckan_cron_jobs.conf b/ckan/supervisor/ckan_cron_jobs.conf index d3268a40..b89069ba 100644 --- a/ckan/supervisor/ckan_cron_jobs.conf +++ b/ckan/supervisor/ckan_cron_jobs.conf @@ -9,7 +9,10 @@ [program:ckan-cron-jobs] ; Use the full paths to the virtualenv and your configuration file here. -command=/usr/sbin/crond -f -L /srv/app/logs/ckan-cron-jobs.log +# This configuration sets up a cron job for CKAN. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini file. +# It then executes the crond command with the -f flag to run in the foreground. +command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/sbin/crond -f" ; User the worker runs as. @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-crons-jobs.stdout.log -stderr_logfile=/srv/app/logs/ckan-crons-jobs.stderr.log +stderr_logfile=/srv/app/logs/ckan-crons-jobs.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_bulk.conf b/ckan/supervisor/ckan_worker_bulk.conf index 1aaa2ed4..02385fcf 100644 --- a/ckan/supervisor/ckan_worker_bulk.conf +++ b/ckan/supervisor/ckan_worker_bulk.conf @@ -9,6 +9,9 @@ [program:ckan-worker-bulk] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker bulk command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_default.conf b/ckan/supervisor/ckan_worker_default.conf index 30378a35..b3ac1861 100644 --- a/ckan/supervisor/ckan_worker_default.conf +++ b/ckan/supervisor/ckan_worker_default.conf @@ -9,6 +9,9 @@ [program:ckan-worker-default] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/ckan/supervisor/ckan_worker_priority.conf b/ckan/supervisor/ckan_worker_priority.conf index bd348274..a7fc1d43 100644 --- a/ckan/supervisor/ckan_worker_priority.conf +++ b/ckan/supervisor/ckan_worker_priority.conf @@ -9,6 +9,9 @@ [program:ckan-worker-priority] ; Use the full paths to the virtualenv and your configuration file here. +# This configuration file specifies the command to run for the CKAN worker bulk process. +# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file. +# It then executes the ckan command with the specified configuration file and runs the jobs worker priority command. command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority" @@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d ; Log files. -stdout_logfile=/srv/app/logs/ckan-worker.stdout.log -stderr_logfile=/srv/app/logs/ckan-worker.stderr.log +stderr_logfile=/srv/app/logs/ckan-worker.log ; Make sure that the worker is started on system start and automatically diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 99a3736a..496ede1c 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -4,6 +4,7 @@ volumes: ckan_storage: pg_data: solr_data: + ckan_logs: services: @@ -32,6 +33,7 @@ services: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions - ./ckan/config:/srv/app/config + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] @@ -62,6 +64,8 @@ services: volumes: - ckan_storage:/var/lib/ckan - ./src:/srv/app/src_extensions + - ./ckan/config:/srv/app/config + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/docker-compose.yml b/docker-compose.yml index 29d12c04..b9be683b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ volumes: ckan_storage: pg_data: solr_data: + ckan_logs: services: @@ -25,6 +26,7 @@ services: - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}" volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs ckan: container_name: ${CKAN_CONTAINER_NAME} @@ -53,6 +55,7 @@ services: condition: service_healthy volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] @@ -87,6 +90,7 @@ services: condition: service_started volumes: - ckan_storage:/var/lib/ckan + - ckan_logs:/srv/app/logs restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"] diff --git a/nginx/setup/default.conf b/nginx/setup/default.conf index c93e5ccd..de344696 100644 --- a/nginx/setup/default.conf +++ b/nginx/setup/default.conf @@ -19,7 +19,10 @@ server { # ssl_session_cache dfine in stream and http ssl_session_tickets off; - #access_log /var/log/nginx/host.access.log main; + access_log /srv/app/logs/nginx_access.log main; + access_log /var/log/nginx/access.log main; + error_log /srv/app/logs/nginx_error.log error; + error_log /var/log/nginx/errror.log error; location / { proxy_pass http://ckan:5000/; diff --git a/nginx/setup/default.conf.template b/nginx/setup/default.conf.template index 00511652..e57fbb50 100644 --- a/nginx/setup/default.conf.template +++ b/nginx/setup/default.conf.template @@ -19,7 +19,10 @@ server { # ssl_session_cache dfine in stream and http ssl_session_tickets off; - #access_log /var/log/nginx/host.access.log main; + access_log /srv/app/logs/nginx_access.log main; + access_log /var/log/nginx/access.log main; + error_log /srv/app/logs/nginx_error.log error; + error_log /var/log/nginx/errror.log error; location / { proxy_pass http://${CKAN_CONTAINER_NAME}:${CKAN_PORT}/;