From fc0b75b2bea187b2ee687f2366813d4b3b7017d0 Mon Sep 17 00:00:00 2001 From: Moritz Breitbach Date: Sun, 17 Dec 2023 00:21:12 +0100 Subject: [PATCH 01/31] Fix typo in nc-format-USB.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Moritz Breitbach Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/TOOLS/nc-format-USB.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp/TOOLS/nc-format-USB.sh b/bin/ncp/TOOLS/nc-format-USB.sh index f3483429a..c6ea8979f 100644 --- a/bin/ncp/TOOLS/nc-format-USB.sh +++ b/bin/ncp/TOOLS/nc-format-USB.sh @@ -51,7 +51,7 @@ configure() } DATADIR="$(get_nc_config_value datadirectory || true)" - if [[ $( stat -fc%d / ) != $( stat -fc%d "$DATADIR" ) ]] || [[ -z "$DATADIR" ]] && [[ "$ALLOW_DATA_DIR_REMOVAL" != "yes" ]] + if [[ $( stat -fc%d / ) != $( stat -fc%d "$DATADIR" ) ]] || [[ -z "$DATADIR" ]] && [[ "$ALLOW_DATADIR_REMOVAL" != "yes" ]] then echo "ERROR: Data directory is on USB drive (or can't be determined) and removal of data directory was not explicitly allowed." \ "Please move the data directory to SD before formatting the USB drive." \ From ee07434b82fa8748429769e565f5df40cc8bca65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 16:35:46 +0200 Subject: [PATCH 02/31] ncp-update-nc: Use systemd-run to run update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 304 +----------------- bin/ncp-update-nc.d/update-nc.sh | 241 ++++++++++++++ .../upgrade-php-bullseye-8.1.sh | 63 ++++ 3 files changed, 315 insertions(+), 293 deletions(-) create mode 100755 bin/ncp-update-nc.d/update-nc.sh create mode 100755 bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 05b1c01f8..ae680d843 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -17,305 +17,23 @@ # - failure at each test point # -> must pass basic NC in tests.py ( meaning it's not in a broken state ) -set -eE${DBG} - -VER="$1" BIN="${0##*/}" - -source /usr/local/etc/library.sh - -# pre-checks -#################### +VER="$1" [[ "$VER" == "" ]] && { echo "Usage ${BIN} "; exit 1; } -[[ -f /.docker-image ]] && BASEDIR=/data || BASEDIR=/var/www -cd "$BASEDIR" -DATADIR="$( get_nc_config_value datadirectory )" -ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Nextcloud is currently down"; exit 1; } -[[ -d "${BASEDIR}/nextcloud-old" ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; } -[[ -d "${BASEDIR}/nextcloud" ]] || { echo "Nextcloud directory not found" ; exit 1; } -[[ -d "$DATADIR" ]] || { echo "Nextcloud data directory not found" ; exit 1; } - -# check version -#################### - -[[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" -CURRENT="$(nc_version)" -TARGET_VERSION="$(determine_nc_upgrade_version "${CURRENT?}" "${VER?}")" -[[ "$TARGET_VERSION" == "$CURRENT" ]] && { - echo "Nextcloud version ${CURRENT} is already installed. Nothing to do." - exit 0 -} -[[ -n "$TARGET_VERSION" ]] || { - echo "Could not find a valid upgrade path from '${CURRENT}' to '${TARGET_VERSION}'. Nothing to update." - exit 1 -} - -MAJOR_NEW="${TARGET_VERSION%%.*}" - -if [[ "$MAJOR_NEW" -ge 24 ]] && [[ "$(lsb_release -r)" =~ .*10 ]] -then - echo -e "NCP doesn't support Nextcloud versions greater than 23 with Debian 10 (Buster). Please run ncp-dist-upgrade." - exit 1 -fi - -if [[ "$MAJOR_NEW" -ge 29 ]] && [[ "$(lsb_release -r)" =~ .*12 ]] -then - echo -e "NCP doesn't support Nextcloud versions greater than 28 with Debian 11 (Bullseye). Please run ncp-dist-upgrade." - exit 1 -fi - -grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; } -grep -qP "\d+\.\d+\.\d+" <<<"$TARGET_VERSION" || { echo "Malformed version $TARGET_VERSION" ; exit 1; } -echo "Current Nextcloud version $CURRENT" -echo "Available Nextcloud version $TARGET_VERSION" -if [[ "$TARGET_VERSION" != "$VER" ]] +if systemctl is-active ncp-update-nc >/dev/null 2>&1 then - echo "INFO: You have requested an update to '$VER', but a direct update to '$VER' cannot be performed, so the latest available version that can be updated to has been selected automatically." -fi - -# make sure that cron.php is not running and there are no pending jobs -# https://github.com/nextcloud/server/issues/10949 -pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; } -pgrep -cf cron.php &>/dev/null && { echo "cron.php running. Abort"; exit 1; } -mysql nextcloud <<<"UPDATE ${DB_PREFIX}jobs SET reserved_at=0;" - -# cleanup -#################### -cleanup() { - local RET=$? - set +eE - echo "Clean up..." - rm -rf "$BASEDIR"/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old - trap "" EXIT - exit $RET -} -trap cleanup EXIT - -# get new code -#################### -URL="https://download.nextcloud.com/server/releases/nextcloud-$TARGET_VERSION.tar.bz2" -echo "Download Nextcloud $TARGET_VERSION..." -wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; } - -# backup -#################### -BKPDIR="$BASEDIR" -WITH_DATA=no -COMPRESSED=yes -LIMIT=0 - -echo "Back up current instance..." -set +eE -ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" # && false # test point -RET=$? -sync -set -eE - -BKP_="$( ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -1 )" -[[ -f "$BKP_" ]] || { set +eE; echo "Error backing up"; false || cleanup; } -[[ $RET -ne 0 ]] && { rm -f "$BKP_"; set +eE; echo "Error backing up"; false || cleanup; } -BKP="$( dirname "$BKP_" )/$( basename "$BKP_" .tar.gz )-${CURRENT}.tar.gz" -echo "Storing backup at '$BKP'..." -mv "$BKP_" "$BKP" - -# simple restore if anything fails from here -#################### -rollback_simple() { - set +eE - trap "" INT TERM HUP ERR - echo -e "Abort\nSimple roll back..." - rm -rf "$BASEDIR"/nextcloud - mv "$BASEDIR"/nextcloud-old "$BASEDIR"/nextcloud - false || cleanup # so cleanup exits with 1 -} -trap rollback_simple INT TERM HUP ERR - -# replace code -#################### -echo "Install Nextcloud $TARGET_VERSION..." -mv -T nextcloud nextcloud-old -tar -xf nextcloud.tar.bz2 # && false # test point -rm -rf /var/www/nextcloud.tar.bz2 - -# copy old config -#################### -cp nextcloud-old/config/config.php nextcloud/config/ - -# copy old themes -#################### -cp -raT nextcloud-old/themes/ nextcloud/themes/ - -# copy old NCP apps -#################### -for app in nextcloudpi previewgenerator; do - if [[ -d nextcloud-old/apps/"${app}" ]]; then - cp -r -L nextcloud-old/apps/"${app}" /var/www/nextcloud/apps/ - fi -done - -#false # test point - -# copy data if it was at the default location -#################### -if [[ "$DATADIR" == "/var/www/nextcloud/data" ]] || [[ "$DATADIR" == "/data/nextcloud/data" ]]; then - echo "Restore data..." - mv -T nextcloud-old/data nextcloud/data + echo "Existing ncp-update-nc process detenced. Connecting..." + tail -f "/var/log/ncp-update-nc.log" + exit 0 fi -# nc-restore if anything fails from here -#################### -rollback() { - set +eE - trap "" INT TERM HUP ERR EXIT - echo -e "Abort\nClean up..." - rm -rf /var/www/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old - echo "Rolling back to backup $BKP..." - local TMPDATA - mkdir -p "$BASEDIR/recovery/" - TMPDATA="$( mktemp -d "$BASEDIR/recovery/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } - [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && mv -T "$DATADIR" "$TMPDATA" - ncp-restore "$BKP" || { echo "Rollback failed! Data left at $TMPDATA"; exit 1; } - [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && { rm -rf "$DATADIR"; mv -T "$TMPDATA" "$DATADIR"; } - rm "$BKP" - echo "Rollback successful. Nothing was updated" - exit 1 -} -trap rollback INT TERM HUP ERR +systemd-run -u 'ncp-update-nc' -p "RemainAfterExit=yes" bash -c "/usr/local/bin/ncp-update-nc.d/update-nc.sh \"${VER}\" |& tee /var/log/ncp-update-nc.log" +sleep 5 -# fix permissions -#################### -echo "Fix permissions..." -chown -R www-data:www-data nextcloud -find nextcloud/ -type d -exec chmod 750 {} \; -find nextcloud/ -type f -exec chmod 640 {} \; - -# upgrade -#################### -echo "Upgrade..." -ncc='sudo -u www-data php nextcloud/occ' -$ncc upgrade # && false # test point -$ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n -$ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n -$ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n -$ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n - -# use the correct version for custom apps -NCVER="$(nc_version)" -if is_more_recent_than "21.0.0" "${NCVER}"; then - NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20 -else - # Install notify_push if not installed - if ! is_app_enabled notify_push; then - ncc app:install notify_push - ncc app:enable notify_push - install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf - a2enmod proxy proxy_http proxy_wstunnel - apachectl -k graceful - ## make sure the notify_push daemon is runnnig - - install_template systemd/notify_push.service.sh /etc/systemd/system/notify_push.service - start_notify_push - nc_domain="$(ncc config:system:get overwrite.cli.url)" - set-nc-domain "${nc_domain}" || { - echo "notify_push setup failed. You are probably behind a proxy" - echo "Run 'ncc config:system:set trusted_proxies 15 --value=' and then 'ncc notify_push:setup https:///push to enable" - echo "Check https://help.nextcloud.com/tags/ncp for support" - } - - fi - NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21 -fi -rm -rf /var/www/nextcloud/apps/previewgenerator -ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator - -if ! is_docker && ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0" +if ! systemctl is-active ncp-update-nc >/dev/null 2>&1 then - ( - echo "Upgrading PHP..." - export DEBIAN_FRONTEND=noninteractive - PHPVER_OLD="$PHPVER" - PHPVER_NEW="8.1" - PHP_PACKAGES_OLD=(php-{common,igbinary,redis,json} "php${PHPVER_OLD}" \ - "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp}) - PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ - "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) - - php_restore() { - trap "" INT TERM HUP ERR - echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." - set +e - service "php${PHPVER_NEW}-fpm" stop - a2disconf php${PHPVER_NEW}-fpm - rm /etc/apt/sources.list.d/php.list - apt-get update - apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" - apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" - set_ncpcfg "php_version" "${PHPVER_OLD}" - install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" - run_app nc-limits - a2enconf "php${PHPVER_OLD}-fpm" - service "php${PHPVER_OLD}-fpm" start - service apache2 restart - echo "PHP upgrade has been successfully reverted" - set -e - } - - trap php_restore INT TERM HUP ERR - - # Setup apt repository for php 8 - wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg - echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list - apt-get update - - clear_opcache - - echo "Stopping apache and php-fpm..." - service "php${PHPVER_OLD}-fpm" stop - service apache2 stop - - echo "Remove old PHP (${PHPVER_OLD})..." - a2disconf "php${PHPVER_OLD}-fpm" - - apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" - - echo "Install PHP ${PHPVER_NEW}..." - install_with_shadow_workaround --no-install-recommends systemd - apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" - - set_ncpcfg "php_version" "${PHPVER_NEW}" - install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" - ( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) - - a2enconf "php${PHPVER_NEW}-fpm" - - echo "Starting apache and php-fpm..." - service "php${PHPVER_NEW}-fpm" start - service apache2 start - ncc status - ) - - # Reload library.sh to reset PHPVER - source /usr/local/etc/library.sh - + echo "Failed to start ncp-update-nc" + systemctl status --no-pager ncp-update-nc fi - -# refresh completions -ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp - -is_docker && { - killall notify_push - sleep 1 - start_notify_push -} - -echo "Update completed successfully." -# done -#################### -mkdir -p "$DATADIR"/ncp-update-backups -mv "$BKP" "$DATADIR"/ncp-update-backups -chown -R www-data:www-data "$DATADIR"/ncp-update-backups -BKP="$DATADIR"/ncp-update-backups/"$( basename "$BKP" )" -echo "Backup stored at $BKP" - -bash -c "sleep 5; source /usr/local/etc/library.sh; clear_opcache;" &>/dev/null & +tail -f "/var/log/ncp-update-nc.log" diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh new file mode 100755 index 000000000..34c9525fa --- /dev/null +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +set -eE${DBG} + +VER="$1" +source /usr/local/etc/library.sh +export RELEASE +export PHPVER + +# pre-checks +#################### +BASEDIR=/var/www +cd "$BASEDIR" +DATADIR="$( get_nc_config_value datadirectory )" +ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Nextcloud is currently down"; exit 1; } +[[ -d "${BASEDIR}/nextcloud-old" ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; } +[[ -d "${BASEDIR}/nextcloud" ]] || { echo "Nextcloud directory not found" ; exit 1; } +[[ -d "$DATADIR" ]] || { echo "Nextcloud data directory not found" ; exit 1; } + +# check version +#################### + +[[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" +CURRENT="$(nc_version)" +TARGET_VERSION="$(determine_nc_upgrade_version "${CURRENT?}" "${VER?}")" +[[ -n "$TARGET_VERSION" ]] || { + echo "Could not find a valid upgrade path from '${CURRENT}' to '${TARGET_VERSION}'. Nothing to update." + exit 1 +} + +MAJOR_NEW="${TARGET_VERSION%%.*}" +DEBIAN_VERSION="$(. /etc/os-release; echo "$VERSION_ID")" + +if [[ "$MAJOR_NEW" -ge 24 ]] && [[ $DEBIAN_VERSION -le 10 ]] +then + echo -e "NCP doesn't support Nextcloud versions greater than 23 with Debian 10 (Buster). Please run ncp-dist-upgrade." + exit 1 +fi + +if [[ "$MAJOR_NEW" -ge 29 ]] && [[ $DEBIAN_VERSION -le 11 ]] +then + echo -e "NCP doesn't support Nextcloud versions greater than 28 with Debian 11 (Bullseye). Please run ncp-dist-upgrade." + exit 1 +fi + +grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; } +grep -qP "\d+\.\d+\.\d+" <<<"$TARGET_VERSION" || { echo "Malformed version $TARGET_VERSION" ; exit 1; } + +echo "Current Nextcloud version $CURRENT" +echo "Available Nextcloud version $TARGET_VERSION" +if [[ "$TARGET_VERSION" != "$VER" ]] +then + echo "INFO: You have requested an update to '$VER', but a direct update to '$VER' cannot be performed, so the latest available version that can be updated to has been selected automatically." +fi + +# make sure that cron.php is not running and there are no pending jobs +# https://github.com/nextcloud/server/issues/10949 +pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; } +pgrep -cf cron.php &>/dev/null && { echo "cron.php running. Abort"; exit 1; } +mysql nextcloud <<<"UPDATE ${DB_PREFIX}jobs SET reserved_at=0;" + +# cleanup +#################### +cleanup() { + local RET=$? + set +eE + echo "Clean up..." + rm -rf "$BASEDIR"/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old + trap "" EXIT + exit $RET +} +trap cleanup EXIT + +# get new code +#################### +URL="https://download.nextcloud.com/server/releases/nextcloud-$TARGET_VERSION.tar.bz2" +echo "Download Nextcloud $TARGET_VERSION..." +wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; } + +# backup +#################### +BKPDIR="$BASEDIR" +WITH_DATA=no +COMPRESSED=yes +LIMIT=0 + +echo "Back up current instance..." +set +eE +ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" # && false # test point +RET=$? +sync +set -eE + +BKP_="$( ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -1 )" +[[ -f "$BKP_" ]] || { set +eE; echo "Error backing up"; false || cleanup; } +[[ $RET -ne 0 ]] && { rm -f "$BKP_"; set +eE; echo "Error backing up"; false || cleanup; } +BKP="$( dirname "$BKP_" )/$( basename "$BKP_" .tar.gz )-${CURRENT}.tar.gz" +echo "Storing backup at '$BKP'..." +mv "$BKP_" "$BKP" + +# simple restore if anything fails from here +#################### +rollback_simple() { + set +eE + trap "" INT TERM HUP ERR + echo -e "Abort\nSimple roll back..." + rm -rf "$BASEDIR"/nextcloud + mv "$BASEDIR"/nextcloud-old "$BASEDIR"/nextcloud + false || cleanup # so cleanup exits with 1 +} +trap rollback_simple INT TERM HUP ERR + +# replace code +#################### +echo "Install Nextcloud $TARGET_VERSION..." +mv -T nextcloud nextcloud-old +tar -xf nextcloud.tar.bz2 # && false # test point +rm -rf /var/www/nextcloud.tar.bz2 + +# copy old config +#################### +cp nextcloud-old/config/config.php nextcloud/config/ + +# copy old themes +#################### +cp -raT nextcloud-old/themes/ nextcloud/themes/ + +# copy old NCP apps +#################### +for app in nextcloudpi previewgenerator; do + if [[ -d nextcloud-old/apps/"${app}" ]]; then + cp -r -L nextcloud-old/apps/"${app}" /var/www/nextcloud/apps/ + fi +done + +#false # test point + +# copy data if it was at the default location +#################### +if [[ "$DATADIR" == "/var/www/nextcloud/data" ]] || [[ "$DATADIR" == "/data/nextcloud/data" ]]; then + echo "Restore data..." + mv -T nextcloud-old/data nextcloud/data +fi + +# nc-restore if anything fails from here +#################### +rollback() { + set +eE + trap "" INT TERM HUP ERR EXIT + echo -e "Abort\nClean up..." + rm -rf /var/www/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old + echo "Rolling back to backup $BKP..." + local TMPDATA + mkdir -p "$BASEDIR/recovery/" + TMPDATA="$( mktemp -d "$BASEDIR/recovery/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } + [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && mv -T "$DATADIR" "$TMPDATA" + ncp-restore "$BKP" || { echo "Rollback failed! Data left at $TMPDATA"; exit 1; } + [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && { rm -rf "$DATADIR"; mv -T "$TMPDATA" "$DATADIR"; } + rm "$BKP" + echo "Rollback successful. Nothing was updated" + exit 1 +} +trap rollback INT TERM HUP ERR + +# fix permissions +#################### +echo "Fix permissions..." +chown -R www-data:www-data nextcloud +find nextcloud/ -type d -exec chmod 750 {} \; +find nextcloud/ -type f -exec chmod 640 {} \; + +# upgrade +#################### +echo "Upgrade..." +ncc='sudo -u www-data php nextcloud/occ' +$ncc upgrade # && false # test point +$ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n +$ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n +$ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n +$ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n + +# use the correct version for custom apps +NCVER="$(nc_version)" +if is_more_recent_than "21.0.0" "${NCVER}"; then + NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20 +else + # Install notify_push if not installed + if ! is_app_enabled notify_push; then + ncc app:install notify_push + ncc app:enable notify_push + install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf + a2enmod proxy proxy_http proxy_wstunnel + apachectl -k graceful + ## make sure the notify_push daemon is runnnig + + install_template systemd/notify_push.service.sh /etc/systemd/system/notify_push.service + start_notify_push + nc_domain="$(ncc config:system:get overwrite.cli.url)" + set-nc-domain "${nc_domain}" || { + echo "notify_push setup failed. You are probably behind a proxy" + echo "Run 'ncc config:system:set trusted_proxies 15 --value=' and then 'ncc notify_push:setup https:///push to enable" + echo "Check https://help.nextcloud.com/tags/ncp for support" + } + + fi + NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21 +fi +rm -rf /var/www/nextcloud/apps/previewgenerator +ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator + +if ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0" +then + /usr/local/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh + + # Reload library.sh to reset PHPVER + source /usr/local/etc/library.sh + export PHPVER + export RELEASE +#elif ! is_more_recent_than "29.0.0" "${NCVER}" && is_more_recent_than "8.3.0" "${PHPVER}.0" && [[ "$DEBIAN_VERSION" -ge 12 ]] +#then +# /usr/local/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh +# +# # Reload library.sh to reset PHPVER +# source /usr/local/etc/library.sh +# export PHPVER +# export RELEASE +fi + +# refresh completions +ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp + +echo "Update completed successfully." +# done +#################### +mkdir -p "$DATADIR"/ncp-update-backups +mv "$BKP" "$DATADIR"/ncp-update-backups +chown -R www-data:www-data "$DATADIR"/ncp-update-backups +BKP="$DATADIR"/ncp-update-backups/"$( basename "$BKP" )" +echo "Backup stored at $BKP" + +bash -c "sleep 5; source /usr/local/etc/library.sh; clear_opcache;" &>/dev/null & diff --git a/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh new file mode 100755 index 000000000..f1ce83d05 --- /dev/null +++ b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +echo "Upgrading PHP..." +export DEBIAN_FRONTEND=noninteractive +PHPVER_OLD="$PHPVER" +PHPVER_NEW="8.1" +PHP_PACKAGES_OLD=(php-{common,igbinary,redis,json} "php${PHPVER_OLD}" \ + "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp}) +PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ + "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) + +php_restore() { + trap "" INT TERM HUP ERR + echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." + set +e + service "php${PHPVER_NEW}-fpm" stop + a2disconf php${PHPVER_NEW}-fpm + rm /etc/apt/sources.list.d/php.list + apt-get update + apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" + set_ncpcfg "php_version" "${PHPVER_OLD}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + run_app nc-limits + a2enconf "php${PHPVER_OLD}-fpm" + service "php${PHPVER_OLD}-fpm" start + service apache2 restart + echo "PHP upgrade has been successfully reverted" + set -e +} + +trap php_restore INT TERM HUP ERR + +# Setup apt repository for php 8 +wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg +echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list +apt-get update + +clear_opcache + +echo "Stopping apache and php-fpm..." +service "php${PHPVER_OLD}-fpm" stop +service apache2 stop + +echo "Remove old PHP (${PHPVER_OLD})..." +a2disconf "php${PHPVER_OLD}-fpm" + +apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" + +echo "Install PHP ${PHPVER_NEW}..." +install_with_shadow_workaround --no-install-recommends systemd +apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" + +set_ncpcfg "php_version" "${PHPVER_NEW}" +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" +( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) + +a2enconf "php${PHPVER_NEW}-fpm" + +echo "Starting apache and php-fpm..." +service "php${PHPVER_NEW}-fpm" start +service apache2 start +ncc status \ No newline at end of file From de9088f4188bb52deae4954671b2a1ba56af454b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 16:49:48 +0200 Subject: [PATCH 03/31] release.yml: Add info about included software versions to release description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b92f3cf95..4c86bf7c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -324,6 +324,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + + NC_VERSION="$(jq -r '.nextcloud_version' < etc/ncp.cfg)" + PHP_VERSION="$(jq -r '.php_version' < etc/ncp.cfg)" + DEBIAN_VERSION="$(jq -r '.release' < etc/ncp.cfg)" + ARMBIAN_VERSIOn="$(cat build/armbian/armbian_version)" + subject="$(git tag -n10 --format="%(contents:subject)" "${{ env.VERSION }}")" body="$(git tag -n30 --format="%(contents:body)" "${{ env.VERSION }}")" separator=" @@ -334,6 +340,12 @@ jobs: gh release create --draft -F - "${{ env.VERSION }}" < Date: Sun, 18 Aug 2024 16:55:34 +0200 Subject: [PATCH 04/31] ncp-update-nc: Propagate debug flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index ae680d843..03cbe5dac 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -17,18 +17,20 @@ # - failure at each test point # -> must pass basic NC in tests.py ( meaning it's not in a broken state ) +set -eE${DBG} + BIN="${0##*/}" VER="$1" [[ "$VER" == "" ]] && { echo "Usage ${BIN} "; exit 1; } if systemctl is-active ncp-update-nc >/dev/null 2>&1 then - echo "Existing ncp-update-nc process detenced. Connecting..." + echo "Existing ncp-update-nc process detected. Connecting..." tail -f "/var/log/ncp-update-nc.log" exit 0 fi -systemd-run -u 'ncp-update-nc' -p "RemainAfterExit=yes" bash -c "/usr/local/bin/ncp-update-nc.d/update-nc.sh \"${VER}\" |& tee /var/log/ncp-update-nc.log" +systemd-run -u 'ncp-update-nc' -p "RemainAfterExit=yes" bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" sleep 5 if ! systemctl is-active ncp-update-nc >/dev/null 2>&1 From 1fb070df550042994fef71462e264653d22f85dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 17:04:12 +0200 Subject: [PATCH 05/31] ncp-update-nc: fix update termination detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 33 ++++++++++++++++++++++++++++----- build/armbian/armbian_version | 1 + build/build-SD-armbian.sh | 2 +- 3 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 build/armbian/armbian_version diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 03cbe5dac..97007e6ed 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -23,19 +23,42 @@ BIN="${0##*/}" VER="$1" [[ "$VER" == "" ]] && { echo "Usage ${BIN} "; exit 1; } -if systemctl is-active ncp-update-nc >/dev/null 2>&1 +connect_to_nc_update() { + tail -f "/var/log/ncp-update-nc.log" & + tail_pid=$! + while [[ "$(systemctl is-active ncp-update-nc ||:)" == "active" ]] + do + sleep 3 + done + + kill "$tail_pid" + if [[ "$(systemctl is-active ncp-update-nc ||:)" == "inactive" ]] + then + echo "Nextcloud update finished successfully." + return 0 + elif [[ "$(systemctl is-active ncp-update-nc ||:)" == "failed" ]] + then + echo "Nextcloud update failed." + return 1 + else + echo "Nextcloud update was not found or failed (unexpected status: '$(systemctl is-active ncp-update-nc ||:)')" + fi +} + +if [[ "$(systemctl is-active ncp-update-nc ||:)" == "activating" ]] then echo "Existing ncp-update-nc process detected. Connecting..." - tail -f "/var/log/ncp-update-nc.log" + connect_to_nc_update exit 0 fi -systemd-run -u 'ncp-update-nc' -p "RemainAfterExit=yes" bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" +systemd-run -u 'ncp-update-nc' bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" sleep 5 -if ! systemctl is-active ncp-update-nc >/dev/null 2>&1 +if [[ "$(systemctl is-active ncp-update-nc ||:)" != "activating" ]] then echo "Failed to start ncp-update-nc" systemctl status --no-pager ncp-update-nc fi -tail -f "/var/log/ncp-update-nc.log" + +connect_to_nc_update diff --git a/build/armbian/armbian_version b/build/armbian/armbian_version new file mode 100644 index 000000000..3e5291183 --- /dev/null +++ b/build/armbian/armbian_version @@ -0,0 +1 @@ +v24.08 \ No newline at end of file diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index 69dae6814..a642411d0 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -31,7 +31,7 @@ prepare_dirs # tmp cache output # get latest armbian [[ -d armbian ]] || { - git clone --depth 1 --branch v24.08 https://github.com/armbian/build armbian + git clone --depth 1 --branch "$(cat "${0}/../armbian/armbian_version")" https://github.com/armbian/build armbian } #( cd armbian && git pull --ff-only --tags && git checkout v23.02 ) #sed -i -e '/export rootfs_size=/s/du -sm/du --apparent-size -sm/' armbian/lib/functions/image/partitioning.sh From 0e30c79a87f9c3b90b1d4fe7f0422c2f6fd5b21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 17:51:17 +0200 Subject: [PATCH 06/31] build-lxd.yml: Pin nc version to update to before dist upgrade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 1d9404856..00f18edd2 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -457,7 +457,7 @@ jobs: echo "Running update to ${VERSION}" current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" - latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" + latest_nc_version="27.0.1" sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" sudo "$LXC" exec ncp -- /usr/local/bin/ncc status @@ -467,6 +467,7 @@ jobs: echo "Nextcloud is up to date - skipping NC update test." else sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status fi sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log @@ -514,7 +515,21 @@ jobs: echo "can't upgrade from Debian $(sudo "$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)" exit 1 } + current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" + sudo "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status + + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + echo "Nextcloud is up to date - skipping NC update test." + else + sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status + fi + + sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log From 73da8fd0577dd15b40b883f89667a7b2998b8d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 17:58:38 +0200 Subject: [PATCH 07/31] ncp.cfg,nc-nextcloud.cfg,ncp-app/: Add support for NC 29 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-config.d/nc-nextcloud.cfg | 2 +- etc/ncp.cfg | 2 +- ncp-app/appinfo/info.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/ncp-config.d/nc-nextcloud.cfg b/etc/ncp-config.d/nc-nextcloud.cfg index 8f7efa697..a3c428d17 100644 --- a/etc/ncp-config.d/nc-nextcloud.cfg +++ b/etc/ncp-config.d/nc-nextcloud.cfg @@ -9,7 +9,7 @@ { "id": "VER", "name": "Version", - "value": "28.0.7" + "value": "29.0.4" }, { "id": "MAXFILESIZE", diff --git a/etc/ncp.cfg b/etc/ncp.cfg index 86fae7fec..4ae0b95c9 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { - "nextcloud_version": "28.0.7", + "nextcloud_version": "29.0.4", "php_version": "8.1", "release": "bookworm" } diff --git a/ncp-app/appinfo/info.xml b/ncp-app/appinfo/info.xml index f182f0487..fbf207ddb 100644 --- a/ncp-app/appinfo/info.xml +++ b/ncp-app/appinfo/info.xml @@ -12,7 +12,7 @@ tools https://github.com/nextcloud/nextcloudpi/issues - + From e05db188974b97e315474e1351186671d04e287c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 17:59:32 +0200 Subject: [PATCH 08/31] update-nc.sh: Implement upgrade of PHP to 8.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 8 +-- bin/ncp-update-nc.d/update-nc.sh | 16 ++--- .../upgrade-php-bookworm-8.3.sh | 64 +++++++++++++++++++ .../upgrade-php-bullseye-8.1.sh | 2 + bin/ncp/CONFIG/nc-init.sh | 10 +-- .../appinfo/info.xml | 2 +- 6 files changed, 82 insertions(+), 20 deletions(-) create mode 100755 bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 97007e6ed..bf5ec1af2 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -26,7 +26,7 @@ VER="$1" connect_to_nc_update() { tail -f "/var/log/ncp-update-nc.log" & tail_pid=$! - while [[ "$(systemctl is-active ncp-update-nc ||:)" == "active" ]] + while [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] do sleep 3 done @@ -45,17 +45,17 @@ connect_to_nc_update() { fi } -if [[ "$(systemctl is-active ncp-update-nc ||:)" == "activating" ]] +if [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] then echo "Existing ncp-update-nc process detected. Connecting..." connect_to_nc_update - exit 0 + exit $? fi systemd-run -u 'ncp-update-nc' bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" sleep 5 -if [[ "$(systemctl is-active ncp-update-nc ||:)" != "activating" ]] +if ! [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] then echo "Failed to start ncp-update-nc" systemctl status --no-pager ncp-update-nc diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh index 34c9525fa..b6b34a040 100755 --- a/bin/ncp-update-nc.d/update-nc.sh +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -214,16 +214,12 @@ then # Reload library.sh to reset PHPVER source /usr/local/etc/library.sh - export PHPVER - export RELEASE -#elif ! is_more_recent_than "29.0.0" "${NCVER}" && is_more_recent_than "8.3.0" "${PHPVER}.0" && [[ "$DEBIAN_VERSION" -ge 12 ]] -#then -# /usr/local/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh -# -# # Reload library.sh to reset PHPVER -# source /usr/local/etc/library.sh -# export PHPVER -# export RELEASE +elif ! is_more_recent_than "29.0.0" "${NCVER}" && is_more_recent_than "8.3.0" "${PHPVER}.0" && [[ "$DEBIAN_VERSION" -ge 12 ]] +then + /usr/local/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh + + # Reload library.sh to reset PHPVER + source /usr/local/etc/library.sh fi # refresh completions diff --git a/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh new file mode 100755 index 000000000..e17e7e21f --- /dev/null +++ b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +source /usr/local/etc/library.sh + +echo "Upgrading PHP..." +export DEBIAN_FRONTEND=noninteractive +PHPVER_OLD="$PHPVER" +PHPVER_NEW="8.3" +PHP_PACKAGES_OLD=("php${PHPVER_OLD}" \ + "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) +PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ + "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) + +php_restore() { + trap "" INT TERM HUP ERR + echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." + set +e + service "php${PHPVER_NEW}-fpm" stop + a2disconf php${PHPVER_NEW}-fpm + wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg + echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list + apt-get update + apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" + set_ncpcfg "php_version" "${PHPVER_OLD}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + run_app nc-limits + a2enconf "php${PHPVER_OLD}-fpm" + service "php${PHPVER_OLD}-fpm" start + service apache2 restart + echo "PHP upgrade has been successfully reverted" + set -e +} + +trap php_restore INT TERM HUP ERR + +apt-get update + +clear_opcache + +echo "Stopping apache and php-fpm..." +service "php${PHPVER_OLD}-fpm" stop +service apache2 stop + +echo "Remove old PHP (${PHPVER_OLD})..." +a2disconf "php${PHPVER_OLD}-fpm" + +apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" + +echo "Install PHP ${PHPVER_NEW}..." +install_with_shadow_workaround --no-install-recommends systemd +apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" + +set_ncpcfg "php_version" "${PHPVER_NEW}" +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" +( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) + +a2enconf "php${PHPVER_NEW}-fpm" + +echo "Starting apache and php-fpm..." +service "php${PHPVER_NEW}-fpm" start +service apache2 start +ncc status +rm -f /etc/apt/sources.list.d/php.list /etc/apt/trusted.gpg.d/php.gpg \ No newline at end of file diff --git a/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh index f1ce83d05..4df0f7a95 100755 --- a/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh +++ b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +source /usr/local/etc/library.sh + echo "Upgrading PHP..." export DEBIAN_FRONTEND=noninteractive PHPVER_OLD="$PHPVER" diff --git a/bin/ncp/CONFIG/nc-init.sh b/bin/ncp/CONFIG/nc-init.sh index 7892b8e2d..0bc62554c 100644 --- a/bin/ncp/CONFIG/nc-init.sh +++ b/bin/ncp/CONFIG/nc-init.sh @@ -127,11 +127,11 @@ EOF } mysql nextcloud < - + From 335df8bd03edb52778c5306c8b8496c541e4f74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 18 Aug 2024 22:19:58 +0200 Subject: [PATCH 09/31] nextcloud_tests.py: Implement workaround for selenium selector issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- tests/nextcloud_tests.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index f4fbe209e..ce0b17857 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -27,7 +27,8 @@ from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.firefox.options import Options -from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException, ElementNotInteractableException from typing import List, Tuple import traceback @@ -129,14 +130,34 @@ def close_first_run_wizard(driver: WebDriver): except NoSuchElementException: pass if first_run_wizard is not None and first_run_wizard.is_displayed(): - wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CLASS_NAME, "modal-container__close"), - (By.CLASS_NAME, "first-run-wizard__close-button")])) + # wait.until(VisibilityOfElementLocatedByAnyLocator([(By.XPATH, "//div[@id='firstrunwizard]//button[@aria-label='Close']'")])) + #print(first_run_wizard.find_element(By.CSS_SELECTOR, 'button[aria-label="Close"]')) try: - overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") - overlay_close_btn.click() - except NoSuchElementException: - overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") - overlay_close_btn.click() + wait.until(VisibilityOfElementLocatedByAnyLocator([(By.XPATH, "//button[@aria-label='Close']"), + (By.CLASS_NAME, "modal-container__close"), + (By.CLASS_NAME, "first-run-wizard__close-button")])) + try: + driver.execute_script(""" + let btn = document.querySelector('#firstrunwizard button[aria-label="Close"]'); + if(btn) { + btn.click(); + } + """) + except (NoSuchElementException, ElementNotInteractableException): + try: + overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") + overlay_close_btn.click() + except (NoSuchElementException, ElementNotInteractableException): + overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") + overlay_close_btn.click() + except TimeoutException: + driver.execute_script(""" + let btn = document.querySelector('#firstrunwizard button[aria-label="Close"]'); + if(btn) { + btn.click(); + } + """) + time.sleep(3) From 4d9102a9670feac8f12ed35f489954ad69c39e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 00:22:31 +0200 Subject: [PATCH 10/31] update-nc.sh: Run maintenance:repair --include-expensive after update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 5 +++-- bin/ncp-update-nc.d/update-nc.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index bf5ec1af2..b96b04126 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -55,10 +55,11 @@ fi systemd-run -u 'ncp-update-nc' bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" sleep 5 -if ! [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] +if ! [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|inactive|activating|deactivating)$ ]] then echo "Failed to start ncp-update-nc" - systemctl status --no-pager ncp-update-nc + [[ -f /var/log/ncp-update-nc.log ]] && cat /var/log/ncp-update-nc.log + systemctl status --no-pager ncp-update-nc ||: fi connect_to_nc_update diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh index b6b34a040..3d197fbc2 100755 --- a/bin/ncp-update-nc.d/update-nc.sh +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -178,6 +178,7 @@ $ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n $ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n $ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n $ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n +$ncc maintenance:repair --help | grep -q -e '--include-expensive' && $ncc maintenance:repair --include-expensive # use the correct version for custom apps NCVER="$(nc_version)" From 51a2d0f66226656cf685bfa56f00b8715f5c476e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 02:09:46 +0200 Subject: [PATCH 11/31] update-nc.sh: Improve output if no new NC version is available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc.d/update-nc.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh index 3d197fbc2..499d1c0bf 100755 --- a/bin/ncp-update-nc.d/update-nc.sh +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -23,6 +23,10 @@ ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Next [[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" CURRENT="$(nc_version)" TARGET_VERSION="$(determine_nc_upgrade_version "${CURRENT?}" "${VER?}")" +[[ "$TARGET_VERSION" == "$CURRENT" ]] && { + echo "Nextcloud version ${CURRENT} is already installed. Nothing to do." + exit 0 +} [[ -n "$TARGET_VERSION" ]] || { echo "Could not find a valid upgrade path from '${CURRENT}' to '${TARGET_VERSION}'. Nothing to update." exit 1 From eddb9d393edff67726ed1c0c02ce531de91a003b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 02:42:14 +0200 Subject: [PATCH 12/31] nextcloudpi/lib/SettingsService.php: Fix running commands in nonexisting working directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp-app/lib/Service/SettingsService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncp-app/lib/Service/SettingsService.php b/ncp-app/lib/Service/SettingsService.php index dfc25d3c5..4db7da964 100644 --- a/ncp-app/lib/Service/SettingsService.php +++ b/ncp-app/lib/Service/SettingsService.php @@ -132,7 +132,7 @@ private function runCommand(string $cmd): array { 2 => ["pipe", "w"] ]; - $proc = proc_open($cmd, $descriptorSpec, $pipes, "/home/www-data", null); + $proc = proc_open($cmd, $descriptorSpec, $pipes, "/var/www", null); $stdout = stream_get_contents($pipes[1]); fclose($pipes[1]); $stderr = stream_get_contents($pipes[2]); From 2e38ecc57e6e847402b3acc07cdf0585e5b9085b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 02:46:06 +0200 Subject: [PATCH 13/31] lamp.sh,ncp.cfg: Use PHP 8.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-sd-images.yml | 6 +++--- bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh | 8 ++++++-- etc/ncp.cfg | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 9aaebb2e5..a44780818 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -191,9 +191,9 @@ jobs: echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf - sudo mkdir -p raspbian_root/etc/systemd/system/php8.1-fpm.service.d - echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf - echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf + sudo mkdir -p raspbian_root/etc/systemd/system/php8.2-fpm.service.d + echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.2-fpm.service.d/ncp.conf + echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.2-fpm.service.d/ncp.conf - name: Test image id: test run: | diff --git a/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh index e17e7e21f..c84c322a8 100755 --- a/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh +++ b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh @@ -57,8 +57,12 @@ install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opc a2enconf "php${PHPVER_NEW}-fpm" +[[ -f "/etc/systemd/system/php${PHPVER_OLD}-fpm.service.d/ncp.conf" ]] && { + mkdir -p "/etc/systemd/system/php${PHPVER_NEW}-fpm.service.d" + cp "/etc/systemd/system/php${PHPVER_OLD}-fpm.service.d/ncp.conf" "/etc/systemd/system/php${PHPVER_NEW}-fpm.service.d/ncp.conf" +} + echo "Starting apache and php-fpm..." service "php${PHPVER_NEW}-fpm" start service apache2 start -ncc status -rm -f /etc/apt/sources.list.d/php.list /etc/apt/trusted.gpg.d/php.gpg \ No newline at end of file +ncc status \ No newline at end of file diff --git a/etc/ncp.cfg b/etc/ncp.cfg index 4ae0b95c9..19b3afcdd 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { "nextcloud_version": "29.0.4", - "php_version": "8.1", + "php_version": "8.3", "release": "bookworm" } From 2838c5901271f51262da815b5ebf46589771c449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 03:12:50 +0200 Subject: [PATCH 14/31] build-lxd.yml: Use compatible repository versions to run tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 00f18edd2..95cb48ede 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -362,7 +362,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - ref: "${{ env.VERSION }}" + ref: "v1.54.3" - name: Setup Firefox continue-on-error: true id: setup-firefox-browser-action @@ -422,7 +422,7 @@ jobs: sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true exit 1 } - python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { + python nextcloud_tests.py --skip-release-check --no-gui "nextcloudpi.local" 443 4443 || { echo "Nextcloud test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true @@ -507,6 +507,10 @@ jobs: echo "System test failed!" exit 1 } + - name: Checkout current version + run: | + git fetch origin + git checkout "${{ env.VERSION }}" - name: NCP distupgrade id: distupgrade run: | From 92587ac8131fd48bd3040cdc5bcf4d5c8955bed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:47:34 +0200 Subject: [PATCH 15/31] README.md: Update referred debian version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c35eebc19..00980c7d0 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Please reach out in the [Matrix][chat-matrix-wiki] or [Telegram][chat-telegram-w ## Features - * Raspberry Pi OS/Debian 11 _(Bullseye)_ + * Raspberry Pi OS/Debian 12 _(Bookworm)_ * Nextcloud * Apache, with HTTP2 enabled * PHP 8.1 From 6598d864fa36f19aa956f02337702fde532f8a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:23:46 +0200 Subject: [PATCH 16/31] nc-encrypt.sh: Run gocryptfs as transient systemd service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-update-nc | 1 + bin/ncp/SECURITY/nc-encrypt.sh | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index b96b04126..1e8ce9151 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -52,6 +52,7 @@ then exit $? fi +systemctl reset-failed ncp-encrypt ||: systemd-run -u 'ncp-update-nc' bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" sleep 5 diff --git a/bin/ncp/SECURITY/nc-encrypt.sh b/bin/ncp/SECURITY/nc-encrypt.sh index 51f5f8b8b..47b6c274f 100644 --- a/bin/ncp/SECURITY/nc-encrypt.sh +++ b/bin/ncp/SECURITY/nc-encrypt.sh @@ -55,9 +55,11 @@ configure() return fi + export PASSWORD # Just mount already encrypted data if [[ -f "${encdir?}"/gocryptfs.conf ]]; then - echo "${PASSWORD?}" | gocryptfs -allow_other -q "${encdir}" "${datadir}" 2>&1 | sed /^Switch/d + systemctl reset-failed ncp-encrypt ||: + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" # switch to the regular virtual hosts after we decrypt, so we can access NC and ncp-web a2ensite ncp 001-nextcloud @@ -75,7 +77,8 @@ configure() mv "${datadir?}" "${tmpdir?}" mkdir "${datadir}" - echo "${PASSWORD}" | gocryptfs -allow_other -q "${encdir}" "${datadir}" 2>&1 | sed /^Switch/d + systemctl reset-failed ncp-encrypt ||: + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" echo "Encrypting data..." mv "${tmpdir}"/* "${tmpdir}"/.[!.]* "${datadir}" From 7612fd9e6b090800ce71adae59787c49a065d584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Tue, 20 Aug 2024 12:35:21 +0200 Subject: [PATCH 17/31] ncp.sh, 1.55.0.sh: Install logrotate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp.sh | 3 ++- updates/1.55.0.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 updates/1.55.0.sh diff --git a/ncp.sh b/ncp.sh index 8813c60cd..c2caedcd3 100644 --- a/ncp.sh +++ b/ncp.sh @@ -22,7 +22,8 @@ install() { # NCP-CONFIG apt-get update - $APTINSTALL git dialog whiptail jq file lsb-release tmux + $APTINSTALL git dialog whiptail jq file lsb-release tmux logrotate + mkdir -p "$CONFDIR" "$BINDIR" # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager diff --git a/updates/1.55.0.sh b/updates/1.55.0.sh new file mode 100644 index 000000000..45033e31f --- /dev/null +++ b/updates/1.55.0.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +apt-get update +apt-get install -y --no-install-recommends logrotate From e7a40c7cc91b4cc5a780e89700dcd35a5d91e5a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:53:48 +0200 Subject: [PATCH 18/31] nextcloud_tests.py: Fix first run wizard close button not being detected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- tests/nextcloud_tests.py | 41 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index ce0b17857..cbc87d83c 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -123,40 +123,31 @@ def is_admin_notifications_checkbox(item: WebElement): def close_first_run_wizard(driver: WebDriver): - wait = WebDriverWait(driver, 60) + wait = WebDriverWait(driver, 20) first_run_wizard = None try: first_run_wizard = driver.find_element(By.CSS_SELECTOR, "#firstrunwizard") except NoSuchElementException: pass if first_run_wizard is not None and first_run_wizard.is_displayed(): - # wait.until(VisibilityOfElementLocatedByAnyLocator([(By.XPATH, "//div[@id='firstrunwizard]//button[@aria-label='Close']'")])) - #print(first_run_wizard.find_element(By.CSS_SELECTOR, 'button[aria-label="Close"]')) + for i in range(3): + try: + wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, '.modal-container__content button[aria-label=Close]'), + (By.CLASS_NAME, "modal-container__close"), + (By.CLASS_NAME, "first-run-wizard__close-button")])) + except TimeoutException as e: + if i == 3: + raise e try: - wait.until(VisibilityOfElementLocatedByAnyLocator([(By.XPATH, "//button[@aria-label='Close']"), - (By.CLASS_NAME, "modal-container__close"), - (By.CLASS_NAME, "first-run-wizard__close-button")])) + overlay_close_btn = driver.find_element(By.CSS_SELECTOR, '.modal-container__content button[aria-label=Close]') + overlay_close_btn.click() + except (NoSuchElementException, ElementNotInteractableException): try: - driver.execute_script(""" - let btn = document.querySelector('#firstrunwizard button[aria-label="Close"]'); - if(btn) { - btn.click(); - } - """) + overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") + overlay_close_btn.click() except (NoSuchElementException, ElementNotInteractableException): - try: - overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") - overlay_close_btn.click() - except (NoSuchElementException, ElementNotInteractableException): - overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") - overlay_close_btn.click() - except TimeoutException: - driver.execute_script(""" - let btn = document.querySelector('#firstrunwizard button[aria-label="Close"]'); - if(btn) { - btn.click(); - } - """) + overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") + overlay_close_btn.click() time.sleep(3) From e5113f4601ba5821e8735030156b8bbc041003ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:20:41 +0200 Subject: [PATCH 19/31] library.sh: Respect requested NC version in determine_nc_update_version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-check-nc-version | 2 +- bin/ncp-update-nc.d/update-nc.sh | 12 +++++++++--- bin/ncp/UPDATES/nc-update-nextcloud.sh | 6 ------ etc/library.sh | 19 ++++++++++++++++--- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/bin/ncp-check-nc-version b/bin/ncp-check-nc-version index a89c63273..972a5b8a4 100755 --- a/bin/ncp-check-nc-version +++ b/bin/ncp-check-nc-version @@ -7,7 +7,7 @@ set -e source /usr/local/etc/library.sh # sets NCLATESTVER CURRENT="$(nc_version)" -NEXT_VERSION="$(determine_nc_upgrade_version "${CURRENT}" "${NCLATESTVER?}")" +NEXT_VERSION="$(determine_nc_update_version "${CURRENT}" "${NCLATESTVER?}")" [[ -n "$NEXT_VERSION" ]] || exit 0 NOTIFIED=/var/run/.nc-version-notified diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh index 499d1c0bf..1055971bf 100755 --- a/bin/ncp-update-nc.d/update-nc.sh +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -22,7 +22,13 @@ ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Next [[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" CURRENT="$(nc_version)" -TARGET_VERSION="$(determine_nc_upgrade_version "${CURRENT?}" "${VER?}")" +if [[ "$VER" == "0" ]] || [[ "$VER" == "" ]] +then + REQUESTED_VERSION="${NCLATESTVER?}" +else + REQUESTED_VERSION="$VER" +fi +TARGET_VERSION="$(determine_nc_update_version "${CURRENT?}" "${NCLATESTVER}" "${REQUESTED_VERSION}")" [[ "$TARGET_VERSION" == "$CURRENT" ]] && { echo "Nextcloud version ${CURRENT} is already installed. Nothing to do." exit 0 @@ -52,9 +58,9 @@ grep -qP "\d+\.\d+\.\d+" <<<"$TARGET_VERSION" || { echo "Malformed version $TA echo "Current Nextcloud version $CURRENT" echo "Available Nextcloud version $TARGET_VERSION" -if [[ "$TARGET_VERSION" != "$VER" ]] +if [[ "$TARGET_VERSION" != "$REQUESTED_VERSION" ]] then - echo "INFO: You have requested an update to '$VER', but a direct update to '$VER' cannot be performed, so the latest available version that can be updated to has been selected automatically." + echo "INFO: You have requested an update to '${REQUESTED_VERSION}', but a direct update to '${REQUESTED_VERSION}' cannot be performed, so the latest available version that can be updated to has been selected automatically." fi # make sure that cron.php is not running and there are no pending jobs diff --git a/bin/ncp/UPDATES/nc-update-nextcloud.sh b/bin/ncp/UPDATES/nc-update-nextcloud.sh index 821bb5e08..58c7487f3 100644 --- a/bin/ncp/UPDATES/nc-update-nextcloud.sh +++ b/bin/ncp/UPDATES/nc-update-nextcloud.sh @@ -12,12 +12,6 @@ LATEST="$NCLATESTVER" configure() { - [[ "$VERSION" == "0" ]] && VERSION="$LATEST" - if ! is_docker && ! is_more_recent_than "24.0.0" "${VERSION}" && is_more_recent_than "8.1.0" "${PHPVER}.0" && [[ " ${BASH_SOURCE[*]} " =~ .*" /home/www/ncp-launcher.sh ".* ]] - then - echo "We need to upgrade PHP. This process cannot be performed from the web UI. Please use 'ncp-config' from the terminal (via SSH or direct access) to update Nextcloud instead. Future updates can again be run from the web UI" - exit 1 - fi bash /usr/local/bin/ncp-update-nc "$VERSION" } diff --git a/etc/library.sh b/etc/library.sh index 1cefac995..c880bbf4d 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -519,16 +519,29 @@ function nc_version() ncc status | grep "versionstring:" | awk '{ print $3 }' } -function determine_nc_upgrade_version() { +function determine_nc_update_version() { local current supported current_maj supported_maj versions next_version current="${1?}" supported="${2?}" + requested="${3:-latest}" #CURRENT="$(ncc status | grep "versionstring:" | awk '{ print $3 }')" current_maj="${current%%.*}" - supported_maj="${supported%%.*}" - versions="$(curl -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/nextcloud/server/releases?per_page=100 | jq -r '.[].tag_name' | grep -v -e 'rc.$' -e 'beta.$' | sort -V)" + requested_maj="${requested%%.*}" + + # If valid version is requested -> direct update, don't consider anything else + if [[ "$requested" =~ ^[0-9.]*$ ]] && [[ "$requested_maj" -le "$((current_maj + 1))" ]] + then + echo "$requested" + return 0 + fi + + versions="$(curl -q -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/nextcloud/server/releases?per_page=100 | jq -r '.[].tag_name' | grep -v -e 'rc.$' -e 'beta.$' | sort -V)" next_version="$(grep "v${current_maj}." <<<"${versions}" | tail -n 1 | tr -d 'v')" + next_version_maj="${next_version%%.*}" + next_version_min="${next_version#*.}" + next_version_min="${next_version_min%%.*}" + if [[ "${next_version}" == "${current}" ]] then From e21a027c684948c52a0c06bc5dde8970c0de677f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:21:10 +0200 Subject: [PATCH 20/31] build-lxd.yml: Use older nextcloud version for upgrade test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 34 +++++++++++++++++++++++++-------- bin/ncp-update-nc | 2 +- etc/library.sh | 1 + 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 95cb48ede..10332fb2f 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -348,8 +348,8 @@ jobs: VERSION: "${{ inputs.git_ref || github.ref }}" LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}" - PREVIOUS_IMAGE_URL_ARM64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_arm64_v1.53.2.tar.gz" - PREVIOUS_IMAGE_URL_AMD64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz" + PREVIOUS_IMAGE_URL_ARM64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextcloudPi_LXD_arm64_v1.53.0.tar.gz" + PREVIOUS_IMAGE_URL_AMD64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextCloudPi_LXD_x86_v1.53.0.tar.gz" USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" steps: - name: Setup incus @@ -457,7 +457,11 @@ jobs: echo "Running update to ${VERSION}" current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" - latest_nc_version="27.0.1" + latest_nc_version="27.1.6" + + sudo "$LXC" exec ncp -- apt-get update ||: + sudo "$LXC" exec ncp -- apt-get install --no-install-recommends -y gnupg2 + sudo "$LXC" exec ncp -- apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" sudo "$LXC" exec ncp -- /usr/local/bin/ncc status @@ -529,14 +533,28 @@ jobs: then echo "Nextcloud is up to date - skipping NC update test." else - sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" - sudo "$LXC" exec ncp -- /usr/local/bin/ncc status + + for i in {1..10}; + do + echo "running nc update ($i/10)..." + sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status + current_nc_version_new="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + break + fi + if [[ "$current_nc_version" == "$current_nc_version_new" ]] + then + echo "failed to update to $latest_nc_version" + exit 1 + fi + current_nc_version="$current_nc_version_new" + done fi sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log - - sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log - sudo "$LXC" stop ncp - name: Relaunch container run: | diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 1e8ce9151..337cc8c84 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -24,7 +24,7 @@ VER="$1" [[ "$VER" == "" ]] && { echo "Usage ${BIN} "; exit 1; } connect_to_nc_update() { - tail -f "/var/log/ncp-update-nc.log" & + tail -n 100 -f "/var/log/ncp-update-nc.log" & tail_pid=$! while [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] do diff --git a/etc/library.sh b/etc/library.sh index c880bbf4d..74eafcb14 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -528,6 +528,7 @@ function determine_nc_update_version() { #CURRENT="$(ncc status | grep "versionstring:" | awk '{ print $3 }')" current_maj="${current%%.*}" requested_maj="${requested%%.*}" + supported_maj="${supported%%.*}" # If valid version is requested -> direct update, don't consider anything else if [[ "$requested" =~ ^[0-9.]*$ ]] && [[ "$requested_maj" -le "$((current_maj + 1))" ]] From 3b65ecf812778eba28f8e3f1c0e81037d3b19364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:00:42 +0200 Subject: [PATCH 21/31] 1.54.0.sh: Prevent apt update from failing due to invalid signing key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- updates/1.54.0.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/updates/1.54.0.sh b/updates/1.54.0.sh index dddcf0327..de15a2ae0 100644 --- a/updates/1.54.0.sh +++ b/updates/1.54.0.sh @@ -26,7 +26,7 @@ done echo "done." echo "Updating PHP package signing key..." -apt-get update +apt-get update ||: apt-get install --no-install-recommends -y gnupg2 apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg From ef849572d7348ffae6add3b6b0f775b46ad90eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:50:14 +0200 Subject: [PATCH 22/31] build-SD-armbian.sh: Fix path to armbian_version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- build/build-SD-armbian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index a642411d0..9e4d97174 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -31,7 +31,7 @@ prepare_dirs # tmp cache output # get latest armbian [[ -d armbian ]] || { - git clone --depth 1 --branch "$(cat "${0}/../armbian/armbian_version")" https://github.com/armbian/build armbian + git clone --depth 1 --branch "$(cat "$(dirname "${0}")/armbian/armbian_version")" https://github.com/armbian/build armbian } #( cd armbian && git pull --ff-only --tags && git checkout v23.02 ) #sed -i -e '/export rootfs_size=/s/du -sm/du --apparent-size -sm/' armbian/lib/functions/image/partitioning.sh From 31d5ed984fb440a7179bee6d236e4c9c01b6f130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 24 Aug 2024 13:10:41 +0200 Subject: [PATCH 23/31] build-sd-images.yml: Adjust php fpm fix to correct php version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-sd-images.yml | 31 ++++++++++++++++--------- build/armbian/armbian.sh | 2 ++ ncp-app/lib/Service/SettingsService.php | 5 +++- ncp.sh | 2 +- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index a44780818..bb518ca54 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -151,12 +151,12 @@ jobs: steps: - name: Set up QEMU uses: docker/setup-qemu-action@v3 -# - name: Set up QEMU -# run: | -# sudo apt-get update -# sudo apt-get install -y binfmt-support -# docker run --rm --privileged tonistiigi/binfmt:latest --install all -# docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes + - name: Set up QEMU + run: | + sudo apt-get update + sudo apt-get install -y binfmt-support + docker run --rm --privileged tonistiigi/binfmt:latest --install all + docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes # sudo systemctl disable apparmor # sudo mkdir -p /etc/binfmt # for conf in qemu-{aarch64,arm}-static.conf @@ -191,9 +191,9 @@ jobs: echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf - sudo mkdir -p raspbian_root/etc/systemd/system/php8.2-fpm.service.d - echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.2-fpm.service.d/ncp.conf - echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.2-fpm.service.d/ncp.conf + sudo mkdir -p raspbian_root/etc/systemd/system/php8.3-fpm.service.d + echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.3-fpm.service.d/ncp.conf + echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.3-fpm.service.d/ncp.conf - name: Test image id: test run: | @@ -208,13 +208,13 @@ jobs: sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" & sleep 60 - CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) + CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) success=false for attempt in {1..30} do echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) ==" - ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" + ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; } success=true break @@ -310,6 +310,14 @@ jobs: python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} Nextcloud test failed!" + echo -e "${LOG_DIAG} /etc/os-release:" + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /etc/os-release' + echo -e "${LOG_DIAG} /usr/local/etc/ncp.cfg:" + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /usr/local/etc/ncp.cfg' + cat ./raspbian_root/usr/local/etc/ncp.cfg + echo -e "${LOG_DIAG} /home/ncp-app-bridge confi g ncp" + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'sudo -u www-data sudo /home/www/ncp-app-bridge.sh config ncp' + sudo ls -l ./raspbian_root/home/www/ncp-app-bridge.sh echo -e "{$LOG_DIAG} Geckodriver logs:" tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo -e "${LOG_CICD} ================" @@ -319,6 +327,7 @@ jobs: echo "${LOG_DIAG} Nextcloud log: " "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + cat ./raspbian_root/opt/ncdata/data/nextcloud.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" sleep 12 continue } diff --git a/build/armbian/armbian.sh b/build/armbian/armbian.sh index 985121062..3d79f4340 100644 --- a/build/armbian/armbian.sh +++ b/build/armbian/armbian.sh @@ -19,6 +19,8 @@ LINUXFAMILY=$2 BOARD=$3 BUILD_DESKTOP=$4 +echo "include home dir? ${INCLUDE_HOME_DIR:-no}" + cd /tmp/overlay NCPCFG=etc/ncp.cfg source etc/library.sh # sets RELEASE diff --git a/ncp-app/lib/Service/SettingsService.php b/ncp-app/lib/Service/SettingsService.php index 4db7da964..bc69fb900 100644 --- a/ncp-app/lib/Service/SettingsService.php +++ b/ncp-app/lib/Service/SettingsService.php @@ -38,6 +38,7 @@ public function getConfig(string $name, array $defaults): array } if ($config == null) { $this->logger->error("Failed to retrieve ncp config (exit code: $ret)"); + $this->logger->error("ERR: $stderr") return $defaults; } return $config; @@ -132,11 +133,13 @@ private function runCommand(string $cmd): array { 2 => ["pipe", "w"] ]; - $proc = proc_open($cmd, $descriptorSpec, $pipes, "/var/www", null); + $proc = proc_open($cmd, $descriptorSpec, $pipes, "/home/www", null); $stdout = stream_get_contents($pipes[1]); fclose($pipes[1]); $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); + + $this->logger->error("STDERR: $stderr") return [proc_close($proc), $stdout, $stderr]; } } diff --git a/ncp.sh b/ncp.sh index c2caedcd3..875dbe3a1 100644 --- a/ncp.sh +++ b/ncp.sh @@ -128,7 +128,7 @@ EOF cat > /home/www/ncp-app-bridge.sh <<'EOF' #!/bin/bash -set -e +set -ex grep -q '[\\&#;`|*?~<>^()[{}$&]' <<< "$*" && exit 1 action="${1?}" [[ "$action" == "config" ]] && { From 8538bfe631553ea6c9d097f8ccdb55f2172ce0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:40:58 +0200 Subject: [PATCH 24/31] build-sd-images.yml: Fix armbian tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-sd-images.yml | 63 +++++++-------------------- 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index bb518ca54..c44083977 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -64,29 +64,28 @@ jobs: # update-binfmts --display qemu-aarch64 # update-binfmts --display qemu-arm - name: "Build Armbian" - if: ${{ inputs.board_id != 'raspberrypi' }} id: build-armbian continue-on-error: true run: | set -ex export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" [[ "${{ github.ref_protected }}" == true ]] || export DBG=x - + ./build/build-SD-armbian.sh "${{ inputs.board_id }}" "${{ inputs.board_name }}" - + artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Build Armbian (2nd attempt)" - if: ${{ inputs.board_id != 'raspberrypi' && steps.build-armbian.outcome == 'failure' }} + if: ${{ steps.build-armbian.outcome == 'failure' }} id: build-armbian-2nd run: | set -ex echo -e "${LOG_CICD} Cleanup armbian build leftovers..." sudo rm -rf armbian/ tmp/ output/ - + export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" [[ "${{ github.ref_protected }}" == true ]] || export DBG=x @@ -98,34 +97,11 @@ jobs: echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Upload Armbian logs" - if: ${{ inputs.board_id != 'raspberrypi' && failure() }} + if: ${{ failure() }} uses: actions/upload-artifact@v3 with: name: ${{ github.run_id }}-${{ inputs.board_id }}-logs path: armbian/output - - name: Build RPI SD Image - if: ${{ inputs.board_id == 'raspberrypi' }} - id: build-rpi - run: | - set -ex - echo -e "${LOG_CICD} Protected? ${{ github.ref_protected }}" - export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" - [[ "${{ github.ref_protected }}" == true ]] || export DBG=x - wget -q https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O ./qemu-aarch64-static - ./build/build-SD-rpi.sh - mkdir -p output - mv "tmp/$IMG" ./output/ - - for i in {1..10} - do - sudo losetup | grep "${IMG}" || break; - [[ "$i" -lt 10 ]] || { echo -e "${LOG_CICD} Timeout while waiting for image to unmount"; exit 1; } - sleep 6 - echo -e "${LOG_CICD} Retrying ($i out of 10)" - done - - echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT - echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: upload image to artifact store uses: actions/upload-artifact@v3 with: @@ -135,7 +111,7 @@ jobs: test: needs: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: VERSION: "${{ inputs.git_ref }}" ARTIFACT_ID: ${{ needs.build.outputs.artifact_name }} @@ -151,18 +127,9 @@ jobs: steps: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Set up QEMU + - name: Disable apparmor run: | - sudo apt-get update - sudo apt-get install -y binfmt-support - docker run --rm --privileged tonistiigi/binfmt:latest --install all - docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes -# sudo systemctl disable apparmor - # sudo mkdir -p /etc/binfmt - # for conf in qemu-{aarch64,arm}-static.conf - # do - # sed 's/:F$/:OC/' /usr/lib/binfmt.d/$conf | sudo tee /etc/binfmt/$conf - # done + sudo systemctl disable apparmor - name: Checkout code uses: actions/checkout@v3 with: @@ -249,12 +216,14 @@ jobs: done [[ "$success" == "true" ]] || { - echo -e "${LOG_CICD} Timeout reached." - "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" + echo -e "${LOG_CICD} ERR: Timeout reached." + "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q systemctl status redis-server |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + sudo journalctl --file ./raspbian_root/var/log/journal/"$(sudo cat ./raspbian_root/etc/machine-id)"/system.journal --no-pager -eu redis-server ||: + "${CONTAINER_CMD[@]}" -q journalctl --no-pager -eu redis-server + "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: exit 1 } From 94dd609fab0c9c96e18549e0b24b4784191673ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:52:16 +0200 Subject: [PATCH 25/31] vm-tests.yml: Fix installer tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-sd-images.yml | 2 +- .github/workflows/vm-tests.yml | 2 ++ tests/nextcloud_tests.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index c44083977..92ef05045 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -296,7 +296,7 @@ jobs: echo "${LOG_DIAG} Nextcloud log: " "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true - cat ./raspbian_root/opt/ncdata/data/nextcloud.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" + sudo cat ./raspbian_root/opt/ncdata/data/nextcloud.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" sleep 12 continue } diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index a1ab770a3..74123b4ce 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -57,6 +57,7 @@ jobs: name: "${{ env.SSH_ARTIFACT_NAME }}" path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error + include-hidden-files: true - id: create-test-instance uses: ./.github/actions/create-test-instance with: @@ -306,6 +307,7 @@ jobs: name: "${{ env.SSH_ARTIFACT_NAME }}" path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error + include-hidden-files: true - id: create-test-instance uses: ./.github/actions/create-test-instance-bullseye with: diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index cbc87d83c..e0bf123e4 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -198,7 +198,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: elif re.match(r'.*Could not check for JavaScript support.*', warning.text): continue # TODO: Solve redis error logs at the source - elif re.match(r'.*\d+ errors in the logs since.*', warning.text): + elif re.match(r'.*\d+ errors? in the logs since.*', warning.text): continue else: raise ConfigTestFailure(f"WARN: {warning.text}") From 325834f86a15b2e2d0e799d0ace025da4c0774b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Sep 2024 19:22:41 +0200 Subject: [PATCH 26/31] SettingsService.php: Fix syntax errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- ncp-app/lib/Service/SettingsService.php | 4 ++-- tests/nextcloud_tests.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ncp-app/lib/Service/SettingsService.php b/ncp-app/lib/Service/SettingsService.php index bc69fb900..444c928ba 100644 --- a/ncp-app/lib/Service/SettingsService.php +++ b/ncp-app/lib/Service/SettingsService.php @@ -38,7 +38,7 @@ public function getConfig(string $name, array $defaults): array } if ($config == null) { $this->logger->error("Failed to retrieve ncp config (exit code: $ret)"); - $this->logger->error("ERR: $stderr") + $this->logger->error("ERR: $stderr"); return $defaults; } return $config; @@ -139,7 +139,7 @@ private function runCommand(string $cmd): array { $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); - $this->logger->error("STDERR: $stderr") + $this->logger->error("STDERR: $stderr"); return [proc_close($proc), $stdout, $stderr]; } } diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index e0bf123e4..dffd4041f 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -122,8 +122,8 @@ def is_admin_notifications_checkbox(item: WebElement): return False -def close_first_run_wizard(driver: WebDriver): - wait = WebDriverWait(driver, 20) +def close_first_run_wizard(driver: WebDriver, wait_multiplier=1): + wait = WebDriverWait(driver, 20 * wait_multiplier) first_run_wizard = None try: first_run_wizard = driver.find_element(By.CSS_SELECTOR, "#firstrunwizard") @@ -152,7 +152,7 @@ def close_first_run_wizard(driver: WebDriver): time.sleep(3) -def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool): +def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool, wait_multiplier=1): """ Login and assert admin page checks""" test = Test() test.new("nextcloud page") @@ -179,7 +179,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: test.report("password", "Wrong password" not in driver.page_source, msg="Failed to login with provided password") test.new("settings config") - wait = WebDriverWait(driver, 60) + wait = WebDriverWait(driver, 60 * wait_multiplier) try: wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, "#security-warning-state-ok"), (By.CSS_SELECTOR, "#security-warning-state-warning"), @@ -236,7 +236,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: except Exception as e: test.check(e) - close_first_run_wizard(driver) + close_first_run_wizard(driver, wait_multiplier) test.new("admin section (1)") try: @@ -289,7 +289,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: except Exception as e: test.check(e) test.new("admin section (2)") - wait = WebDriverWait(driver, 10) + wait = WebDriverWait(driver, 10 * wait_multiplier) try: li = next(filter(is_admin_notifications_checkbox, list_items)) li.find_element(By.TAG_NAME, "input").click() @@ -321,13 +321,14 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: # parse options try: - opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check']) + opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check', 'wait-multiplier=']) except getopt.GetoptError: usage() sys.exit(2) skip_release_check = False options = webdriver.FirefoxOptions() + wait_multiplier = 1 for opt, arg in opts: if opt in ('-h', '--help'): usage() @@ -339,6 +340,8 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: options.add_argument("-headless") elif opt == '--skip-release-check': skip_release_check = True + elif opt == '--wait-multiplier': + wait_multiplier = int(arg) else: usage() sys.exit(2) @@ -377,7 +380,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: driver = webdriver.Firefox(options=options) failed=False try: - test_nextcloud(IP, nc_port, driver, skip_release_check) + test_nextcloud(IP, nc_port, driver, skip_release_check, wait_multiplier) except Exception as e: print(e) print(traceback.format_exc()) From 90ef9dba5384e426919dd4d08bc08a096ed416ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Sep 2024 20:22:32 +0200 Subject: [PATCH 27/31] vm-tests.yml: Fix distupgrade tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .../create-test-instance-bullseye/action.yml | 2 +- .../actions/create-test-instance/action.yml | 2 +- .github/workflows/build-lxd.yml | 2 +- .github/workflows/vm-tests.yml | 47 +++++++++++++++++-- ncp-app/lib/Service/SettingsService.php | 1 - 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/.github/actions/create-test-instance-bullseye/action.yml b/.github/actions/create-test-instance-bullseye/action.yml index 07a8bff2e..dd82d8118 100644 --- a/.github/actions/create-test-instance-bullseye/action.yml +++ b/.github/actions/create-test-instance-bullseye/action.yml @@ -13,7 +13,7 @@ inputs: server_type: description: Server type to use for hetzner servers required: true - default: "cx11" + default: "cx22" outputs: server_address: diff --git a/.github/actions/create-test-instance/action.yml b/.github/actions/create-test-instance/action.yml index ff6cdb9c8..26b1a7303 100644 --- a/.github/actions/create-test-instance/action.yml +++ b/.github/actions/create-test-instance/action.yml @@ -13,7 +13,7 @@ inputs: server_type: description: Server type to use for hetzner servers required: true - default: "cx11" + default: "cx22" outputs: server_address: diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 10332fb2f..c5b2992a1 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -541,7 +541,7 @@ jobs: sudo "$LXC" exec ncp -- /usr/local/bin/ncc status current_nc_version_new="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" - if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]] then break fi diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index 74123b4ce..a6cf114a6 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -64,7 +64,7 @@ jobs: version: ${{ env.VERSION }} uid: "${{ env.UID }}" hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} - server_type: "cx11" + server_type: "cx22" - name: set instance variables run: | echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" @@ -314,7 +314,7 @@ jobs: version: "${{ env.PREVIOUS_VERSION }}" uid: "${{ env.UID }}" hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} - server_type: "cx11" + server_type: "cx22" - name: Set instance variables run: | echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" @@ -341,7 +341,7 @@ jobs: echo "Run integration tests" ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg - test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + test-ncp-instance -a -f "$SNAPSHOT_ID" -b "v1.54.3" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" @@ -387,7 +387,7 @@ jobs: NC_TEST_ARGS=() ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check") set -x - test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + test-ncp-instance -f "$SNAPSHOT_ID" -b "v1.54.3" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" @@ -420,6 +420,45 @@ jobs: } ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade echo "skipped=no" | tee -a $GITHUB_OUTPUT + - name: Update Nextcloud + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin + run: | + set -ex + apk add jq + + echo "Setup ssh" + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + current_nc_version="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')" + latest_nc_version="$(cat ../../etc/ncp.cfg | jq -r '.nextcloud_version')" + + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + echo "Nextcloud is up to date - skipping NC update test." + else + for i in {1..10}; + do + echo "running nc update ($i/10)..." + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "bash -c 'DBG=x ncp-update-nc ${latest_nc_version?}'" + + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" /usr/local/bin/ncc status + + current_nc_version_new="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')" + if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]] + then + break + fi + if [[ "$current_nc_version" == "$current_nc_version_new" ]] + then + echo "failed to update to $latest_nc_version" + exit 1 + fi + current_nc_version="$current_nc_version_new" + done + fi - name: Run integration tests after dist-upgrade id: final_test diff --git a/ncp-app/lib/Service/SettingsService.php b/ncp-app/lib/Service/SettingsService.php index 444c928ba..daa78d46a 100644 --- a/ncp-app/lib/Service/SettingsService.php +++ b/ncp-app/lib/Service/SettingsService.php @@ -139,7 +139,6 @@ private function runCommand(string $cmd): array { $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); - $this->logger->error("STDERR: $stderr"); return [proc_close($proc), $stdout, $stderr]; } } From 1acf1b4a464741a70982956ee1c27d83161bbd43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 6 Sep 2024 14:50:18 +0200 Subject: [PATCH 28/31] release.yml: Add orangepi5 and orangepi5-plus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4c86bf7c8..b6ce6a4f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,6 +136,22 @@ jobs: board_id: odroidc2 board_name: OdroidC2 secrets: inherit + orangepi5: + if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + board_id: orangepi5 + board_name: OrangePi5 + secrets: inherit + orangepi5plus: + if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + board_id: orangepi5-plus + board_name: OrangePi5Plus + secrets: inherit lxd-x86-release: needs: @@ -294,6 +310,30 @@ jobs: artifact_file: "${{ needs.odroidc2.outputs.artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} secrets: inherit + orangepi5-release: + needs: + - orangepi5 + - github-release + if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.orangepi5.outputs.artifact_name }}" + artifact_file: "${{ needs.orangepi5.outputs.artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + secrets: inherit + orangepi5plus-release: + needs: + - orangepi5plus + - github-release + if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.orangepi5plus.outputs.artifact_name }}" + artifact_file: "${{ needs.orangepi5plus.outputs.artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + secrets: inherit github-release: if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(github.ref_name, 'docker-') }} From 93f76ec8f8f065866ebb6b369f856074841a8094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:50:03 +0200 Subject: [PATCH 29/31] changelog.md: Document v1.55.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- changelog.md | 23 +++++++++++++++++++++++ generate_cohortes.sh | 2 ++ staged_rollouts/v1.55.0.txt | 10 ++++++++++ 3 files changed, 35 insertions(+) create mode 100644 staged_rollouts/v1.55.0.txt diff --git a/changelog.md b/changelog.md index 81ea9eddb..09409ce8c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,28 @@ # NextcloudPi Changelog +## [v1.55.0](https://github.com/nextcloud/nextcloudpi/tree/v1.55.0) (2024-09-06) Support for PHP 8.3 and Nextcloud 29 + +### Changes + +- Support for Nextcloud 29 +- Support for PHP 8.3 +- Run `maintenance:repair --include-expensive` after NC update if necessary +- Add logrotate for redis, apache and mysql logs +- Add support for the OrangePi 5 and OrangePi 5 Plus boards + +### Fixes + +- Fix bug in nc-format-USB.sh (thanks @m-breitbach) +- Nextcloud updater fixes + - Fix constant retries of the auto update if on a version that's later than the last officially supported one [#1965](https://github.com/nextcloud/nextcloudpi/issues/1965) + - Fix incorrect detection of os constraints for NC updates beyond v28 [#1960](https://github.com/nextcloud/nextcloudpi/issues/1960) + - Fix user requested version being ignored if a later version was available +- Improve stability of nc-encrypt by daemonizing the encryption process [#1874](https://github.com/nextcloud/nextcloudpi/issues/1874) + +### Development related + +- Perform nc update tests for both lxd as well as vms during CI/CD + ## [v1.54.3](https://github.com/nextcloud/nextcloudpi/tree/v1.54.3) (2024-08-11) Improved Nextcloud (auto) updates ### Fixes diff --git a/generate_cohortes.sh b/generate_cohortes.sh index 83fc87b4a..42bd3bd29 100755 --- a/generate_cohortes.sh +++ b/generate_cohortes.sh @@ -5,6 +5,8 @@ target_file="${2?}" all=({100..1}) cohortes=() +touch "${target_file?}" + while IFS="" read -r line || [ -n "$line" ] do [[ -n "$line" ]] || continue diff --git a/staged_rollouts/v1.55.0.txt b/staged_rollouts/v1.55.0.txt new file mode 100644 index 000000000..8126544f9 --- /dev/null +++ b/staged_rollouts/v1.55.0.txt @@ -0,0 +1,10 @@ +4 +6 +9 +10 +35 +43 +46 +75 +84 +92 \ No newline at end of file From 0e408673472f50b0c60e569a3bc06b0735c054cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:51:44 +0200 Subject: [PATCH 30/31] nc-encrypt.sh: Fix detection of running encryption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-sd-images.yml | 12 +++++------ bin/ncp/SECURITY/nc-encrypt.sh | 30 ++++++++++++++++++++++----- ncp.sh | 2 +- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 92ef05045..12a4490d0 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -282,20 +282,20 @@ jobs: echo -e "${LOG_DIAG} /etc/os-release:" "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /etc/os-release' echo -e "${LOG_DIAG} /usr/local/etc/ncp.cfg:" - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /usr/local/etc/ncp.cfg' + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'cat /usr/local/etc/ncp.cfg' cat ./raspbian_root/usr/local/etc/ncp.cfg - echo -e "${LOG_DIAG} /home/ncp-app-bridge confi g ncp" - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'sudo -u www-data sudo /home/www/ncp-app-bridge.sh config ncp' + echo -e "${LOG_DIAG} /home/ncp-app-bridge config ncp" sudo ls -l ./raspbian_root/home/www/ncp-app-bridge.sh + "${CONTAINER_CMD[@]}" --pipe --uid=33 ncp /bin/bash -c 'sudo /home/www/ncp-app-bridge.sh config ncp' echo -e "{$LOG_DIAG} Geckodriver logs:" tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo -e "${LOG_CICD} ================" echo -e "${LOG_DIAG} ncp.log: " - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo "================" echo "${LOG_DIAG} Nextcloud log: " - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true sudo cat ./raspbian_root/opt/ncdata/data/nextcloud.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" sleep 12 continue diff --git a/bin/ncp/SECURITY/nc-encrypt.sh b/bin/ncp/SECURITY/nc-encrypt.sh index 47b6c274f..efed4d9bb 100644 --- a/bin/ncp/SECURITY/nc-encrypt.sh +++ b/bin/ncp/SECURITY/nc-encrypt.sh @@ -20,14 +20,14 @@ install() configure() { -( + set -e -o pipefail local datadir parentdir encdir tmpdir datadir="$(get_ncpcfg datadir)" [[ "${datadir?}" == "null" ]] && datadir=/var/www/nextcloud/data parentdir="$(dirname "${datadir}")" encdir="${parentdir?}/ncdata_enc" - tmpdir="$(mktemp -u -p "${parentdir}" -t nc-data-crypt.XXXXXX))" + tmpdir="$(mktemp -u -p "${parentdir}" -t nc-data-crypt.XXXXXX)" [[ "${ACTIVE?}" != "yes" ]] && { if ! is_active; then @@ -59,7 +59,7 @@ configure() # Just mount already encrypted data if [[ -f "${encdir?}"/gocryptfs.conf ]]; then systemctl reset-failed ncp-encrypt ||: - systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -fg -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" # switch to the regular virtual hosts after we decrypt, so we can access NC and ncp-web a2ensite ncp 001-nextcloud @@ -72,13 +72,33 @@ configure() mkdir -p "${encdir?}" echo "${PASSWORD?}" | gocryptfs -init -q "${encdir}" save_maintenance_mode + cleanup() { + umount "${datadir}" ||: + [[ -f "${tmpdir}" ]] && { + rm -rf "${datadir?}" ||: + mv "${tmpdir}" "${datadir}" + + chown -R www-data:www-data "${datadir}" + } + } + + trap cleanup 1 trap restore_maintenance_mode EXIT mv "${datadir?}" "${tmpdir?}" mkdir "${datadir}" systemctl reset-failed ncp-encrypt ||: - systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -fg -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" + + maxtries=5 + while [[ "$(systemctl is-active ncp-encrypt)" != "active" ]] || ! mount | grep -1 "${datadir}" + do + echo "Wating for encryption process to start... (${maxtries})" + sleep 3 + maxtries=$((maxtries - 1)) + [[ $maxtries -gt 0 ]] || return 1 + done echo "Encrypting data..." mv "${tmpdir}"/* "${tmpdir}"/.[!.]* "${datadir}" @@ -88,7 +108,7 @@ configure() set_ncpcfg datadir "${datadir}" echo "Data is now encrypted" -) + } # License diff --git a/ncp.sh b/ncp.sh index 875dbe3a1..c2caedcd3 100644 --- a/ncp.sh +++ b/ncp.sh @@ -128,7 +128,7 @@ EOF cat > /home/www/ncp-app-bridge.sh <<'EOF' #!/bin/bash -set -ex +set -e grep -q '[\\&#;`|*?~<>^()[{}$&]' <<< "$*" && exit 1 action="${1?}" [[ "$action" == "config" ]] && { From d52b8dcdf147a15b0e9c553682a2fb12a8374128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sat, 7 Sep 2024 10:42:53 +0200 Subject: [PATCH 31/31] changelog.md: Adjust date for v1.55.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 09409ce8c..0ad4f656d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,6 @@ # NextcloudPi Changelog -## [v1.55.0](https://github.com/nextcloud/nextcloudpi/tree/v1.55.0) (2024-09-06) Support for PHP 8.3 and Nextcloud 29 +## [v1.55.0](https://github.com/nextcloud/nextcloudpi/tree/v1.55.0) (2024-09-07) Support for PHP 8.3 and Nextcloud 29 ### Changes