diff --git a/Dockerfile b/Dockerfile index 70c5e9d9..93f697df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ENV MW_VERSION=REL1_39 \ MW_CORE_VERSION=1.39.7 \ WWW_ROOT=/var/www/mediawiki \ MW_HOME=/var/www/mediawiki/w \ + MW_LOG=/var/log/mediawiki \ MW_ORIGIN_FILES=/mw_origin_files \ MW_VOLUME=/mediawiki \ WWW_USER=www-data \ @@ -84,6 +85,7 @@ RUN set -x; \ && a2enmod proxy_fcgi \ # Create directories && mkdir -p $MW_HOME \ + && mkdir -p $MW_LOG \ && mkdir -p $MW_ORIGIN_FILES \ && mkdir -p $MW_VOLUME @@ -223,6 +225,7 @@ COPY _sources/configs/mpm_event.conf /etc/apache2/mods-available/mpm_event.conf RUN set -x; \ chmod -v +x /*.sh \ + && chmod -v +x /maintenance-scripts/*.sh \ # Sitemap directory && ln -s $MW_VOLUME/sitemap $MW_HOME/sitemap \ # Comment out ErrorLog and CustomLog parameters, we use rotatelogs in mediawiki.conf for the log files diff --git a/_sources/canasta/getMediawikiSettings.php b/_sources/canasta/getMediawikiSettings.php index 2aaee3cf..808e162f 100644 --- a/_sources/canasta/getMediawikiSettings.php +++ b/_sources/canasta/getMediawikiSettings.php @@ -1,6 +1,7 @@ > "$logfileNow" # Job types that need to be run ASAP no matter how many of them are in the queue # Those jobs should be very "cheap" to run - php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1 sleep 1 - php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1 sleep 1 - php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1 sleep 1 - php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1 sleep 1 # Everything else, limit the number of jobs on each batch # The --wait parameter will pause the execution here until new jobs are added, # to avoid running the loop without anything to do - php $RJ --maxjobs=10 --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --maxjobs=10 --server="https://$wiki_url" --wiki="$wiki_id" >> "$logfileNow" 2>&1 # Wait some seconds to let the CPU do other things, like handling web requests, etc - echo mwjobrunner waits for "$MW_JOB_RUNNER_PAUSE" seconds... + echo mwjobrunner waits for "$MW_JOB_RUNNER_PAUSE" seconds... >> "$logfileNow" sleep "$MW_JOB_RUNNER_PAUSE" done } & diff --git a/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh b/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh index fed66e8d..2db8b1cb 100755 --- a/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh +++ b/_sources/scripts/maintenance-scripts/mw_sitemap_generator.sh @@ -1,53 +1,61 @@ #!/bin/bash -# read variables from LocalSettings.php -if [ -z "$MW_SCRIPT_PATH" ]; then - MW_SCRIPT_PATH=$(php /getMediawikiSettings.php --variable=wgScriptPath --format=string) -fi -if [ -z "$MW_SCRIPT_PATH" ]; then - MW_SCRIPT_PATH="/w" -fi +. /functions.sh SCRIPT=$MW_HOME/maintenance/generateSitemap.php +logfileName=mwsitemapgen_log + # Verify the delay is >= 1, otherwise fall back to 1 if [ "$MW_SITEMAP_PAUSE_DAYS" -lt "1" ]; then MW_SITEMAP_PAUSE_DAYS=1 fi # Convert to seconds (suffixed sleep command has issues on OSX) -SLEEPDAYS=$(expr $MW_SITEMAP_PAUSE_DAYS \* 60 \* 60 \* 24) +SLEEP_DAYS=$((MW_SITEMAP_PAUSE_DAYS * 60 * 60 * 24)) -SITE_SERVER=$MW_SITE_SERVER +SITE_SERVER=$(get_mediawiki_variable wgServer) # Fallback to https:// scheme if it's protocol-relative if [[ $SITE_SERVER == "//"* ]]; then SITE_SERVER="https:$SITE_SERVER" fi +SCRIPT_PATH=$(get_mediawiki_variable wgScriptPath) + # Adds slash to sitemap dir if it's not empty and has no starting slash SITEMAP_DIR=$MW_SITEMAP_SUBDIR if [[ -n "$SITEMAP_DIR" && "$SITEMAP_DIR" != "/"* ]]; then SITEMAP_DIR="/$SITEMAP_DIR" fi -GOOGLE_PING_URL="https://www.google.com/ping?sitemap=${SITE_SERVER}${MW_SCRIPT_PATH}/sitemap${SITEMAP_DIR}/sitemap-index-${MW_SITEMAP_IDENTIFIER}.xml" +GOOGLE_PING_URL="https://www.google.com/ping?sitemap=${SITE_SERVER}${SCRIPT_PATH}/sitemap${SITEMAP_DIR}/sitemap-index-${MW_SITEMAP_IDENTIFIER}.xml" -echo Starting sitemap generator... +echo "Starting sitemap generator (in 30 seconds)..." # Wait three minutes after the server starts up to give other processes time to get started sleep 30 echo Sitemap generator started. while true; do - php $SCRIPT \ - --fspath=$MW_HOME/sitemap/$MW_SITEMAP_SUBDIR \ - --urlpath=$MW_SCRIPT_PATH/sitemap/$MW_SITEMAP_SUBDIR \ + logFilePrev="$logfileNow" + logfileNow="$MW_LOG/$logfileName"_$(date +%Y%m%d) + if [ -n "$logFilePrev" ] && [ "$logFilePrev" != "$logfileNow" ]; then + /rotatelogs-compress.sh "$logfileNow" "$logFilePrev" & + fi + + date >> "$logfileNow" + + # generate the sitemap + php "$SCRIPT" \ + --fspath="$MW_HOME/sitemap/$MW_SITEMAP_SUBDIR" \ + --urlpath="$SCRIPT_PATH/sitemap/$MW_SITEMAP_SUBDIR" \ --compress yes \ - --server=$MW_SITE_SERVER \ + --server="$MW_SITE_SERVER" \ --skip-redirects \ - --identifier=$MW_SITEMAP_IDENTIFIER + --identifier="$MW_SITEMAP_IDENTIFIER" \ + >> "$logfileNow" 2>&1 # sending the sitemap to google echo "sending to Google -> $GOOGLE_PING_URL" curl --silent "$GOOGLE_PING_URL" > /dev/null # Wait some seconds to let the CPU do other things, like handling web requests, etc - echo mwsitemapgen waits for "$SLEEPDAYS" seconds... - sleep "$SLEEPDAYS" + echo mwsitemapgen waits for "$SLEEP_DAYS" seconds... >> "$logfileNow" + sleep "$SLEEP_DAYS" done diff --git a/_sources/scripts/maintenance-scripts/mw_transcoder.sh b/_sources/scripts/maintenance-scripts/mw_transcoder.sh index 789a0731..c60a5f85 100755 --- a/_sources/scripts/maintenance-scripts/mw_transcoder.sh +++ b/_sources/scripts/maintenance-scripts/mw_transcoder.sh @@ -1,8 +1,9 @@ #!/bin/bash RJ=$MW_HOME/maintenance/runJobs.php +logfileName=mwtranscoder_log -echo Starting transcoder... +echo "Starting transcoder (in 180 seconds)..." # Wait three minutes after the server starts up to give other processes time to get started sleep 180 @@ -20,12 +21,19 @@ if [ -f "$MW_VOLUME/config/wikis.yaml" ]; then echo "$wiki_id transcoder started." { while true; do - php $RJ --type=webVideoTranscodePrioritized --maxjobs=10 --wiki="$wiki_id" --server="https://$wiki_url" + logFilePrev="$logfileNow" + logfileNow="$MW_LOG/$logfileName"_$(date +%Y%m%d) + if [ -n "$logFilePrev" ] && [ "$logFilePrev" != "$logfileNow" ]; then + /rotatelogs-compress.sh "$logfileNow" "$logFilePrev" & + fi + + date >> "$logfileNow" + php "$RJ" --type webVideoTranscodePrioritized --maxjobs=10 >> "$logfileNow" 2>&1 sleep 1 - php $RJ --type=webVideoTranscode --maxjobs=1 --wiki="$wiki_id" --server="https://$wiki_url" + php "$RJ" --type webVideoTranscode --maxjobs=1 >> "$logfileNow" 2>&1 # Wait some seconds to let the CPU do other things, like handling web requests, etc - echo mwtranscoder waits for "$MW_JOB_TRANSCODER_PAUSE" seconds... + echo mwtranscoder waits for "$MW_JOB_TRANSCODER_PAUSE" seconds... >> "$logfileNow" sleep "$MW_JOB_TRANSCODER_PAUSE" done } & diff --git a/_sources/scripts/run-all.sh b/_sources/scripts/run-all.sh index 1e47024f..db5ae124 100755 --- a/_sources/scripts/run-all.sh +++ b/_sources/scripts/run-all.sh @@ -4,6 +4,14 @@ set -x . /functions.sh +if ! mountpoint -q -- "$MW_VOLUME"; then + echo "Folder $MW_VOLUME contains important data and must be mounted to persistent storage!" + if ! isTrue "$MW_ALLOW_UNMOUNTED_VOLUME"; then + exit 1 + fi + echo "You allowed to continue because MW_ALLOW_UNMOUNTED_VOLUME is set as true" +fi + # Symlink all extensions and skins (both bundled and user) /create-symlinks.sh @@ -13,8 +21,8 @@ set -x # $MW_VOLUME (./extensions, ./skins, ./config, ./images), # note that this command will also set all the necessary permissions echo "Syncing files..." -rsync -ah --inplace --ignore-existing --remove-source-files \ - -og --chown=$WWW_GROUP:$WWW_USER --chmod=Fg=rw,Dg=rwx \ +rsync -ah --inplace --ignore-existing \ + -og --chown="$WWW_GROUP:$WWW_USER" --chmod=Fg=rw,Dg=rwx \ "$MW_ORIGIN_FILES"/ "$MW_VOLUME"/ # We don't need it anymore @@ -30,20 +38,27 @@ rm -rf "$MW_ORIGIN_FILES" # hence it does not perform any recursive checks and may lead to files # or directories down the tree having incorrect permissions left untouched -echo "Checking permissions of $MW_VOLUME..." -if dir_is_writable $MW_VOLUME; then - echo "Permissions are OK!" +# Write log files to $MW_VOLUME/log directory if target folders are not mounted +echo "Checking permissions of Apache log dir $APACHE_LOG_DIR..." +if ! mountpoint -q -- "$APACHE_LOG_DIR/"; then + mkdir -p "$MW_VOLUME/log/httpd" + rsync -avh --ignore-existing "$APACHE_LOG_DIR/" "$MW_VOLUME/log/httpd/" + mv "$APACHE_LOG_DIR" "${APACHE_LOG_DIR}_old" + ln -s "$MW_VOLUME/log/httpd" "$APACHE_LOG_DIR" else - chown -R "$WWW_GROUP":"$WWW_GROUP" "$MW_VOLUME" - chmod -R g=rwX "$MW_VOLUME" + chgrp -R "$WWW_GROUP" "$APACHE_LOG_DIR" + chmod -R g=rwX "$APACHE_LOG_DIR" fi -echo "Checking permissions of $APACHE_LOG_DIR..." -if dir_is_writable $APACHE_LOG_DIR; then - echo "Permissions are OK!" +echo "Checking permissions of PHP-FPM log dir $PHP_LOG_DIR..." +if ! mountpoint -q -- "$PHP_LOG_DIR/"; then + mkdir -p "$MW_VOLUME/log/php-fpm" + rsync -avh --ignore-existing "$PHP_LOG_DIR/" "$MW_VOLUME/log/php-fpm/" + mv "$PHP_LOG_DIR" "${PHP_LOG_DIR}_old" + ln -s "$MW_VOLUME/log/php-fpm" "$PHP_LOG_DIR" else - chown -R "$WWW_GROUP":"$WWW_GROUP" $APACHE_LOG_DIR - chmod -R g=rwX $APACHE_LOG_DIR + chgrp -R "$WWW_GROUP" "$PHP_LOG_DIR" + chmod -R g=rwX "$PHP_LOG_DIR" fi config_subdir_wikis() { @@ -99,12 +114,16 @@ echo "Starting services..." touch "$WWW_ROOT/.maintenance" /run-maintenance-scripts.sh & -echo "Checking permissions of $MW_VOLUME/sitemap..." -if dir_is_writable "$MW_VOLUME/sitemap"; then - echo "Permissions are OK!" +echo "Checking permissions of Mediawiki log dir $MW_LOG..." +if ! mountpoint -q -- "$MW_LOG"; then + mkdir -p "$MW_VOLUME/log/mediawiki" + rsync -avh --ignore-existing "$MW_LOG/" "$MW_VOLUME/log/mediawiki/" + mv "$MW_LOG" "${MW_LOG}_old" + ln -s "$MW_VOLUME/log/mediawiki" "$MW_LOG" + chmod -R o=rwX "$MW_VOLUME/log/mediawiki" else - chown -R "$WWW_GROUP":"$WWW_GROUP" $MW_VOLUME/sitemap - chmod -R g=rwX $MW_VOLUME/sitemap + chgrp -R "$WWW_GROUP" "$MW_LOG" + chmod -R go=rwX "$MW_LOG" fi echo "Checking permissions of MediaWiki volume dir $MW_VOLUME except $MW_VOLUME/images..."