diff --git a/.hooks/pre-push b/.hooks/pre-push index 1746632eeb..e14312bac7 100755 --- a/.hooks/pre-push +++ b/.hooks/pre-push @@ -48,6 +48,10 @@ for arg in "$@"; do esac done +cd install +TESTING=true pipenv run ./test.sh +cd - + echo "Running pre push hook!" repository_path=$(git rev-parse --show-toplevel) tag_name=$(git tag --points-at=HEAD | head -n 1) diff --git a/core/tools/blueos_startup_update/blueos_startup_update b/core/tools/blueos_startup_update/blueos_startup_update index ec338743a7..3c35ff6f14 100755 --- a/core/tools/blueos_startup_update/blueos_startup_update +++ b/core/tools/blueos_startup_update/blueos_startup_update @@ -1,13 +1,19 @@ #!/usr/bin/env python -import appdirs import copy -import os +import datetime import json +import logging +import os import re +import subprocess import sys import time -from typing import Tuple, List +from typing import List, Tuple + +import appdirs +is_testing_environment = os.getenv("TESTING") +SUDO = "" if is_testing_environment else "sudo" # Any change made in this DELTA_JSON dict should be also made # into /bootstrap/startup.json.default too! DELTA_JSON = { @@ -57,8 +63,34 @@ DELTA_JSON = { # However, it is important to note that conflicting configurations can happen, potentially impacting the kernel's loading process or causing harm to BlueOS. CONFIG_USER_PROTECTION_WORD = 'custom' -import collections -from commonwealth.utils.commands import run_command + +if not is_testing_environment: + from commonwealth.utils.commands import run_command + +else: + def run_command(command, check=True): + logging.info(command) + return subprocess.run( + command, + check=check, + text=True, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + +# let's do some logging so we know what happened in previous boots... +now = datetime.datetime.now() + +LOG_PATH = "./" if is_testing_environment else "/var/logs/blueos/services/blueos_startup_update" + +if not os.path.exists(LOG_PATH): + os.makedirs(LOG_PATH) + +log_filename = f"{LOG_PATH}/blueos_startup_update_{now.strftime('%Y%m%d_%H%M%S')}.log" + +logging.basicConfig(filename=log_filename, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.info('Starting the script') # Copyright 2016-2022 Paul Durivage # Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,11 +103,14 @@ def dict_merge(dct, merge_dct): dct[k] = merge_dct[k] def update_startup() -> bool: + if os.getenv('PYTEST'): + logging.info("Testing enviroment dectect, skipping startup.json update") + return False startup_path = os.path.join(appdirs.user_config_dir('bootstrap'), 'startup.json') config = {} if not os.path.isfile(startup_path): - print(f"File: {startup_path}, does not exist, aborting.") + logging.info(f"File: {startup_path}, does not exist, aborting.") return False with open(startup_path, mode="r", encoding="utf-8") as startup_file: @@ -161,27 +196,27 @@ def boot_config_filter_conflicting_configuration_at_session(config_content: List def load_file(file_name) -> str: command = f'cat "{file_name}"' output = run_command(command, False) - print(output) + logging.info(output) return output.stdout def save_file(file_name: str, file_content: str, backup_identifier: str) -> None: - command = f'sudo cp "{file_name}" "{file_name}.{backup_identifier}.bak"' - print(run_command(command, False)) + command = f'{SUDO} cp "{file_name}" "{file_name}.{backup_identifier}.bak"' + logging.info(run_command(command, False)) - command = f'echo "{file_content}" | sudo tee "{file_name}"' - print(run_command(command, False)) + command = f'echo "{file_content}" | {SUDO} tee "{file_name}"' + logging.info(run_command(command, False)) def hardlink_exists(file_name: str) -> bool: command = f"[ -f '{file_name}' ] && [ $(stat -c '%h' '{file_name}') -gt 1 ]" output = run_command(command, False) - print(output) + logging.info(output) return output.returncode == 0 def create_hard_link(source_file_name: str, destination_file_name: str) -> bool: - command = f"sudo rm -rf {destination_file_name}; sudo ln {source_file_name} {destination_file_name}" + command = f"{SUDO} rm -rf {destination_file_name}; {SUDO} ln {source_file_name} {destination_file_name}" output = run_command(command, False) - print(output) + logging.info(output) return output.returncode == 0 @@ -242,10 +277,10 @@ def boot_cmdfile_add_config(cmdline_content: List[str], config_key: str, config_ cmdline_content.append(config_line) def update_cgroups() -> bool: - print("Running cgroup update..") + logging.info("Running cgroup update..") # Retrieve the cmdline file - cmdline_file = '/boot/cmdline.txt' + cmdline_file = os.getenv('CMDLINE_FILE') or '/boot/cmdline.txt' cmdline_content = load_file(cmdline_file).replace('\n','').split(' ') unpatched_cmdline_content = cmdline_content.copy() @@ -271,10 +306,10 @@ def update_cgroups() -> bool: return True def update_dwc2() -> bool: - print("Running dwc2 update..") + logging.info("Running dwc2 update..") # Retrieve the config file - config_file = '/boot/config.txt' + config_file = os.getenv('CONFIG_FILE') or '/boot/config.txt' config_content = load_file(config_file).splitlines() unpatched_config_content = config_content.copy() @@ -294,7 +329,7 @@ def update_dwc2() -> bool: save_file(config_file, config_content_str, backup_identifier) # Retrieve the cmdline file - cmdline_file = '/boot/cmdline.txt' + cmdline_file = os.getenv('CMDLINE_FILE') or '/boot/cmdline.txt' cmdline_content = load_file(cmdline_file).replace('\n','').split(' ') unpatched_cmdline_content = cmdline_content.copy() @@ -313,10 +348,10 @@ def update_dwc2() -> bool: return True def update_navigator_overlays() -> bool: - print("Running Nagivator overlays update..") + logging.info("Running Nagivator overlays update..") # Retrieve the config file - config_file = '/boot/config.txt' + config_file = os.getenv('CONFIG_FILE') or '/boot/config.txt' config_content = load_file(config_file).splitlines() unpatched_config_content = config_content.copy() @@ -372,7 +407,7 @@ def create_dns_conf_host_link() -> bool: # Creates a static reoslv conf to allow docker binds if not create_hard_link(original_resolv_conf_file, resolv_conf_file_host_link): - print("Failed to apply patch") + logging.warning("Failed to apply patch") return False # Patch applied and system needs to be restarted for it to take effect @@ -381,8 +416,8 @@ def create_dns_conf_host_link() -> bool: def ensure_nginx_permissions() -> bool: # ensure nginx can read the userdata directory - command = "sudo chown -R www-data:www-data /usr/blueos/userdata" - print(run_command(command, False)) + command = "{SUDO} chown -R www-data:www-data /usr/blueos/userdata" + logging.info(run_command(command, False)) # This patch doesn't require restart to take effect return False @@ -390,12 +425,12 @@ def ensure_nginx_permissions() -> bool: def ensure_user_data_structure_is_in_place() -> bool: # ensures we have all base folders in userdata commands = [ - "sudo mkdir -p /usr/blueos/userdata/images/vehicle", - "sudo mkdir -p /usr/blueos/userdata/images/logo", - "sudo mkdir -p /usr/blueos/userdata/styles", + "{SUDO} mkdir -p /usr/blueos/userdata/images/vehicle", + "{SUDO} mkdir -p /usr/blueos/userdata/images/logo", + "{SUDO} mkdir -p /usr/blueos/userdata/styles", ] for command in commands: - print(run_command(command, False)) + logging.info(run_command(command, False)) # This patch doesn't require restart to take effect return False @@ -403,7 +438,7 @@ def ensure_user_data_structure_is_in_place() -> bool: def run_command_is_working(): output = run_command("uname -a", check=False) if output.returncode != 0: - print(output) + logging.info(output) return False return True @@ -413,11 +448,11 @@ def main() -> int: current_git_version = os.getenv('GIT_DESCRIBE_TAGS') match = re.match(r'(?P.*)-(?P\d+)-(?P[a-z0-9]+)', current_git_version) tag, commit_number, commit_hash = match['tag'], match['commit_number'], match['commit_hash'] - print(f"Running BlueOS: {tag=}, {commit_number=}, {commit_hash=}") + logging.info(f"Running BlueOS: {tag=}, {commit_number=}, {commit_hash=}") if not run_command_is_working(): - print("Critical error: Something is wrong with the host computer, run_command is not working.") - print("Ignoring host computer configuration for now.") + logging.info("Critical error: Something is wrong with the host computer, run_command is not working.") + logging.info("Ignoring host computer configuration for now.") return 0 # TODO: parse tag as semver and check before applying patches @@ -426,22 +461,30 @@ def main() -> int: update_cgroups, update_dwc2, update_navigator_overlays, - ensure_user_data_structure_is_in_place, - ensure_nginx_permissions, - create_dns_conf_host_link, ] + # only run the next paches on deployment environment + if not is_testing_environment: + patches_to_apply.extend([ + create_dns_conf_host_link, + ensure_nginx_permissions, + ensure_user_data_structure_is_in_place, + ]) - print("The following patches will be applied if needed:", [patch_to_apply.__name__ for patch_to_apply in patches_to_apply]) + logging.info("The following patches will be applied if needed:") + logging.info([patch_to_apply.__name__ for patch_to_apply in patches_to_apply]) patches_requiring_restart = [patch.__name__ for patch in patches_to_apply if patch()] - if patches_requiring_restart: - print("The system will restart in 10 seconds because the following applied patches required restart:", patches_requiring_restart) - time.sleep(10) - run_command('sudo reboot', False) - time.sleep(600) # we are already rebooting anyway. but we don't want the other services to come up - print(f"All patches applied successfully in { time.time() - start} seconds") - return 0 + logging.info(f"All patches applied successfully in { time.time() - start} seconds") + if patches_requiring_restart: + logging.warning("The system will now restart in 10 seconds because the following applied patches required restart:") + logging.warning(patches_requiring_restart) + if not is_testing_environment: + time.sleep(10) + run_command('{SUDO} reboot', False) + time.sleep(600) + return len(patches_requiring_restart) if __name__ == "__main__": - main() + # will return with non-zero if any patch requires reboot + exit(main()) diff --git a/install/boards/bcm_27xx.sh b/install/boards/bcm_27xx.sh index c61dea33c3..a62f78cd72 100755 --- a/install/boards/bcm_27xx.sh +++ b/install/boards/bcm_27xx.sh @@ -6,8 +6,8 @@ VERSION="${VERSION:-master}" GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-bluerobotics/blueos-docker} REMOTE="${REMOTE:-https://raw.githubusercontent.com/${GITHUB_REPOSITORY}}" ROOT="$REMOTE/$VERSION" -CMDLINE_FILE=/boot/cmdline.txt -CONFIG_FILE=/boot/config.txt +CMDLINE_FILE="${CMDLINE_FILE:-/boot/cmdline.txt}" +CONFIG_FILE="${CONFIG_FILE:-/boot/config.txt}" alias curl="curl --retry 6 --max-time 15 --retry-all-errors" # Download, compile, and install spi0 mosi-only device tree overlay for @@ -64,24 +64,45 @@ for STRING in \ sed -i "$line_number r /dev/stdin" $CONFIG_FILE <<< "$STRING" done -# Check for valid modules file to load kernel modules -if [ -f "/etc/modules" ]; then - MODULES_FILE="/etc/modules" -else - MODULES_FILE="/etc/modules-load.d/blueos.conf" - touch "$MODULES_FILE" || true # Create if it does not exist -fi +# Do not run these in CI +if [ -z "$TESTING" ]; then + # Check for valid modules file to load kernel modules + if [ -f "/etc/modules" ]; then + MODULES_FILE="/etc/modules" + else + MODULES_FILE="/etc/modules-load.d/blueos.conf" + touch "$MODULES_FILE" || true # Create if it does not exist + fi -echo "- Set up kernel modules." -# Remove any configuration or commented part related to the i2c drive -for STRING in "bcm2835-v4l2" "i2c-bcm2835" "i2c-dev"; do - sudo sed -i "/$STRING/d" "$MODULES_FILE" - echo "$STRING" | sudo tee -a "$MODULES_FILE" -done + echo "- Set up kernel modules." + # Remove any configuration or commented part related to the i2c drive + for STRING in "bcm2835-v4l2" "i2c-bcm2835" "i2c-dev"; do + sed -i "/$STRING/d" "$MODULES_FILE" + echo "$STRING" | tee -a "$MODULES_FILE" + done + + # Update raspberry pi firmware + # this is required to avoid 'i2c transfer timed out' kernel errors + # on older firmware versions + if grep -q ID=raspbian < /etc/os-release; then + RPI_FIRMWARE_VERSION=1340be4 + if JUST_CHECK=1 rpi-update $RPI_FIRMWARE_VERSION | grep "Firmware update required"; then + echo "- Run rpi update." + SKIP_WARNING=1 rpi-update $RPI_FIRMWARE_VERSION + else + echo "- Firmware is up to date." + fi + fi + + # Force update of bootloader and VL085 firmware on the first boot + echo "- Force update of VL085 and bootloader on first boot." + SYSTEMD_EEPROM_UPDATE_FILE="/lib/systemd/system/rpi-eeprom-update.service" + sed -i '/^ExecStart=\/usr\/bin\/rpi-eeprom-update -s -a$/c\ExecStart=/bin/bash -c "/usr/bin/rpi-eeprom-update -a -d | (grep \\\"reboot to apply\\\" && echo \\\"Rebooting..\\\" && reboot || exit 0)"' $SYSTEMD_EEPROM_UPDATE_FILE +fi # Remove any console serial configuration echo "- Configure serial." -sudo sed -e 's/console=serial[0-9],[0-9]*\ //' -i $CMDLINE_FILE +sed -e 's/console=serial[0-9],[0-9]*\ //' -i $CMDLINE_FILE # Set cgroup, necessary for docker access to memory information echo "- Enable cgroup with memory and cpu" @@ -95,21 +116,3 @@ grep -q dwc2 $CMDLINE_FILE || ( # Append cgroups on the first line sed -i '1 s/$/ modules-load=dwc2,g_ether/' $CMDLINE_FILE ) - -# Update raspberry pi firmware -# this is required to avoid 'i2c transfer timed out' kernel errors -# on older firmware versions -if grep -q ID=raspbian < /etc/os-release; then - RPI_FIRMWARE_VERSION=1340be4 - if sudo JUST_CHECK=1 rpi-update $RPI_FIRMWARE_VERSION | grep "Firmware update required"; then - echo "- Run rpi update." - sudo SKIP_WARNING=1 rpi-update $RPI_FIRMWARE_VERSION - else - echo "- Firmware is up to date." - fi -fi - -# Force update of bootloader and VL085 firmware on the first boot -echo "- Force update of VL085 and bootloader on first boot." -SYSTEMD_EEPROM_UPDATE_FILE="/lib/systemd/system/rpi-eeprom-update.service" -sudo sed -i '/^ExecStart=\/usr\/bin\/rpi-eeprom-update -s -a$/c\ExecStart=/bin/bash -c "/usr/bin/rpi-eeprom-update -a -d | (grep \\\"reboot to apply\\\" && echo \\\"Rebooting..\\\" && reboot || exit 0)"' $SYSTEMD_EEPROM_UPDATE_FILE \ No newline at end of file diff --git a/install/test.sh b/install/test.sh new file mode 100755 index 0000000000..f0e3bf13b2 --- /dev/null +++ b/install/test.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +export TESTING=true +export CONFIG_FILE=tests/bullseye/config.txt +export CMDLINE_FILE=tests/bullseye/cmdline.txt +export GIT_DESCRIBE_TAGS=1.2.0-79-gf5280f32 + + +reset_files() { + git checkout $CONFIG_FILE + git checkout $CMDLINE_FILE +} + + +reset_files +# this is expected to return a non-zero exit code and change both files +if python ../core/tools/blueos_startup_update/blueos_startup_update; then + echo "Error: blueos_startup_update was expected to return a non-zero exit code." + exit 1 +fi + +reset_files + +# this is expected to return a zero exit code and change both files +./boards/bcm_27xx.sh + +# this is expected to return a zero exit code and NOT change any files +GIT_DESCRIBE_TAGS=1.2.0-79-gf5280f32 python ../core/tools/blueos_startup_update/blueos_startup_update + +reset_files diff --git a/install/tests/bullseye/cmdline.txt b/install/tests/bullseye/cmdline.txt new file mode 100644 index 0000000000..f4e1cd1d12 --- /dev/null +++ b/install/tests/bullseye/cmdline.txt @@ -0,0 +1 @@ +console=serial0,115200 console=tty1 root=PARTUUID=e72b8fd1-02 rootfstype=ext4 fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh diff --git a/install/tests/bullseye/config.txt b/install/tests/bullseye/config.txt new file mode 100644 index 0000000000..6e072cb999 --- /dev/null +++ b/install/tests/bullseye/config.txt @@ -0,0 +1,80 @@ +# For more options and information see +# http://rpf.io/configtxt +# Some settings may impact device functionality. See link above for details + +# uncomment if you get no picture on HDMI for a default "safe" mode +#hdmi_safe=1 + +# uncomment the following to adjust overscan. Use positive numbers if console +# goes off screen, and negative if there is too much border +#overscan_left=16 +#overscan_right=16 +#overscan_top=16 +#overscan_bottom=16 + +# uncomment to force a console size. By default it will be display's size minus +# overscan. +#framebuffer_width=1280 +#framebuffer_height=720 + +# uncomment if hdmi display is not detected and composite is being output +#hdmi_force_hotplug=1 + +# uncomment to force a specific HDMI mode (this will force VGA) +#hdmi_group=1 +#hdmi_mode=1 + +# uncomment to force a HDMI mode rather than DVI. This can make audio work in +# DMT (computer monitor) modes +#hdmi_drive=2 + +# uncomment to increase signal to HDMI, if you have interference, blanking, or +# no display +#config_hdmi_boost=4 + +# uncomment for composite PAL +#sdtv_mode=2 + +#uncomment to overclock the arm. 700 MHz is the default. +#arm_freq=800 + +# Uncomment some or all of these to enable the optional hardware interfaces +#dtparam=i2c_arm=on +#dtparam=i2s=on +#dtparam=spi=on + +# Uncomment this to enable infrared communication. +#dtoverlay=gpio-ir,gpio_pin=17 +#dtoverlay=gpio-ir-tx,gpio_pin=18 + +# Additional overlays and parameters are documented /boot/overlays/README + +# Enable audio (loads snd_bcm2835) +dtparam=audio=on + +# Automatically load overlays for detected cameras +camera_auto_detect=1 + +# Automatically load overlays for detected DSI displays +display_auto_detect=1 + +# Enable DRM VC4 V3D driver +dtoverlay=vc4-kms-v3d +max_framebuffers=2 + +# Disable compensation for displays with overscan +disable_overscan=1 + +[cm4] +# Enable host mode on the 2711 built-in XHCI USB controller. +# This line should be removed if the legacy DWC2 controller is required +# (e.g. for USB device mode) or if USB support is not required. +otg_mode=1 + +[all] + +[pi4] +# Run as fast as firmware / board allows +arm_boost=1 + +[all]