From 5340a2207bf55144d5083e9f6939bec3bb018f5b Mon Sep 17 00:00:00 2001 From: TechieNK Date: Fri, 21 Jun 2024 21:50:47 +0530 Subject: [PATCH 1/2] addition of the --memory-limit flag (with a value of 512M) when calling runJobs.php --- Dockerfile | 1 + _sources/canasta/CanastaDefaultSettings.php | 3 ++ .../maintenance-scripts/mw_job_runner.sh | 8 ++-- _sources/scripts/run-all.sh | 38 ++++++------------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9de54bc..dcb65ade 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,6 +177,7 @@ COPY --from=source $MW_ORIGIN_FILES $MW_ORIGIN_FILES # Default values ENV MW_ENABLE_JOB_RUNNER=true \ MW_JOB_RUNNER_PAUSE=2 \ + MW_JOB_RUNNER_MEMORY_LIMIT=512M \ MW_ENABLE_TRANSCODER=true \ MW_JOB_TRANSCODER_PAUSE=60 \ MW_MAP_DOMAIN_TO_DOCKER_GATEWAY=true \ diff --git a/_sources/canasta/CanastaDefaultSettings.php b/_sources/canasta/CanastaDefaultSettings.php index 08ebfc07..276ea67a 100644 --- a/_sources/canasta/CanastaDefaultSettings.php +++ b/_sources/canasta/CanastaDefaultSettings.php @@ -72,6 +72,9 @@ $wgCdnServersNoPurge[] = '172.16.0.0/12'; // 172.16.0.0 – 172.31.255.255 $wgCdnServersNoPurge[] = '192.168.0.0/16'; // 192.168.0.0 – 192.168.255.255 +# SemanticMediaWiki +$smwgConfigFileDir = "$DOCKER_MW_VOLUME/extensions/SemanticMediaWiki/config"; + # Include user defined CommonSettings.php file if ( file_exists( $canastaCommonSettingsFilePath ) ) { require_once "$canastaCommonSettingsFilePath"; diff --git a/_sources/scripts/maintenance-scripts/mw_job_runner.sh b/_sources/scripts/maintenance-scripts/mw_job_runner.sh index 32993344..33e58437 100755 --- a/_sources/scripts/maintenance-scripts/mw_job_runner.sh +++ b/_sources/scripts/maintenance-scripts/mw_job_runner.sh @@ -22,13 +22,13 @@ if [ -f "$MW_VOLUME/config/wikis.yaml" ]; then while true; do # Job types that need to be run ASAP no matter how many of them are in the queue # Those jobs should be very "cheap" to run - php $RJ --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="enotifNotify" --server="https://$wiki_url" --wiki="$wiki_id" sleep 1 - php $RJ --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="createPage" --server="https://$wiki_url" --wiki="$wiki_id" sleep 1 - php $RJ --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="refreshLinks" --server="https://$wiki_url" --wiki="$wiki_id" sleep 1 - php $RJ --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" + php $RJ --memory-limit="$MW_JOB_RUNNER_MEMORY_LIMIT" --type="htmlCacheUpdate" --maxjobs=500 --server="https://$wiki_url" --wiki="$wiki_id" sleep 1 # Everything else, limit the number of jobs on each batch # The --wait parameter will pause the execution here until new jobs are added, diff --git a/_sources/scripts/run-all.sh b/_sources/scripts/run-all.sh index 56b757ab..9a92ac4e 100755 --- a/_sources/scripts/run-all.sh +++ b/_sources/scripts/run-all.sh @@ -24,31 +24,13 @@ isFalse() { esac } -dir_is_writable() { - # Use -L to get information about the target of a symlink, - # not the link itself, as pointed out in the comments - INFO=( $(stat -L -c "0%a %G %U" "$1") ) - PERM=${INFO[0]} - GROUP=${INFO[1]} - OWNER=${INFO[2]} - - if (( ($PERM & 0002) != 0 )); then - # Everyone has write access - return 0 - elif (( ($PERM & 0020) != 0 )); then - # Some group has write access. - # Is user in that group? - if [[ $GROUP == $WWW_GROUP ]]; then - return 0 - fi - elif (( ($PERM & 0200) != 0 )); then - # The owner has write access. - # Does the user own the file? - [[ $WWW_USER == $OWNER ]] && return 0 - fi - - return 1 -} +if ! mountpoint -q -- "$MW_VOLUME"; then + echo "Folder $MW_VOLUME contains important data and must be mounted to persistent storage!" + if ! isTrue "$MW_ALLOW_UNMOUNTED_VOLUME"; then + exit 1 + fi + echo "You allowed to continue because MW_ALLOW_UNMOUNTED_VOLUME is set as true" +fi # Symlink all extensions and skins (both bundled and user) /create-symlinks.sh @@ -63,8 +45,10 @@ rsync -ah --inplace --ignore-existing --remove-source-files \ -og --chown=$WWW_GROUP:$WWW_USER --chmod=Fg=rw,Dg=rwx \ "$MW_ORIGIN_FILES"/ "$MW_VOLUME"/ -# We don't need it anymore -rm -rf "$MW_ORIGIN_FILES" +# Create needed directories +#TODO check below command need +mkdir -p "$MW_VOLUME"/extensions/SemanticMediaWiki/config +mkdir -p "$MW_VOLUME"/l10n_cache /update-docker-gateway.sh From b9365082d627f4b69aa99604495c2ba9686dc0d4 Mon Sep 17 00:00:00 2001 From: TechieNK Date: Sat, 22 Jun 2024 14:24:42 +0530 Subject: [PATCH 2/2] Remove unrelated changes --- _sources/canasta/CanastaDefaultSettings.php | 3 -- _sources/scripts/run-all.sh | 38 +++++++++++++++------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/_sources/canasta/CanastaDefaultSettings.php b/_sources/canasta/CanastaDefaultSettings.php index 276ea67a..08ebfc07 100644 --- a/_sources/canasta/CanastaDefaultSettings.php +++ b/_sources/canasta/CanastaDefaultSettings.php @@ -72,9 +72,6 @@ $wgCdnServersNoPurge[] = '172.16.0.0/12'; // 172.16.0.0 – 172.31.255.255 $wgCdnServersNoPurge[] = '192.168.0.0/16'; // 192.168.0.0 – 192.168.255.255 -# SemanticMediaWiki -$smwgConfigFileDir = "$DOCKER_MW_VOLUME/extensions/SemanticMediaWiki/config"; - # Include user defined CommonSettings.php file if ( file_exists( $canastaCommonSettingsFilePath ) ) { require_once "$canastaCommonSettingsFilePath"; diff --git a/_sources/scripts/run-all.sh b/_sources/scripts/run-all.sh index 9a92ac4e..56b757ab 100755 --- a/_sources/scripts/run-all.sh +++ b/_sources/scripts/run-all.sh @@ -24,13 +24,31 @@ isFalse() { esac } -if ! mountpoint -q -- "$MW_VOLUME"; then - echo "Folder $MW_VOLUME contains important data and must be mounted to persistent storage!" - if ! isTrue "$MW_ALLOW_UNMOUNTED_VOLUME"; then - exit 1 - fi - echo "You allowed to continue because MW_ALLOW_UNMOUNTED_VOLUME is set as true" -fi +dir_is_writable() { + # Use -L to get information about the target of a symlink, + # not the link itself, as pointed out in the comments + INFO=( $(stat -L -c "0%a %G %U" "$1") ) + PERM=${INFO[0]} + GROUP=${INFO[1]} + OWNER=${INFO[2]} + + if (( ($PERM & 0002) != 0 )); then + # Everyone has write access + return 0 + elif (( ($PERM & 0020) != 0 )); then + # Some group has write access. + # Is user in that group? + if [[ $GROUP == $WWW_GROUP ]]; then + return 0 + fi + elif (( ($PERM & 0200) != 0 )); then + # The owner has write access. + # Does the user own the file? + [[ $WWW_USER == $OWNER ]] && return 0 + fi + + return 1 +} # Symlink all extensions and skins (both bundled and user) /create-symlinks.sh @@ -45,10 +63,8 @@ rsync -ah --inplace --ignore-existing --remove-source-files \ -og --chown=$WWW_GROUP:$WWW_USER --chmod=Fg=rw,Dg=rwx \ "$MW_ORIGIN_FILES"/ "$MW_VOLUME"/ -# Create needed directories -#TODO check below command need -mkdir -p "$MW_VOLUME"/extensions/SemanticMediaWiki/config -mkdir -p "$MW_VOLUME"/l10n_cache +# We don't need it anymore +rm -rf "$MW_ORIGIN_FILES" /update-docker-gateway.sh