diff --git a/README.md b/README.md index a52b1de..6aa0cbb 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,22 @@ This action is a composition of [actions/cache](https://github.com/actions/cache Create a workflow `.yml` file in your repositories `.github/workflows` directory. An [example workflow](#example-workflow) is available below. For more information, reference the GitHub Help Documentation for [Creating a workflow file](https://help.github.com/en/articles/configuring-a-workflow#creating-a-workflow-file). +### Versions + +There are three kinds of version labels you can use. + +* `@latest` - This will give you the latest release. +* `@v#` - Major only will give you the latest release for that major version only (e.g. `v1`). +* Branch + * `@master` - Most recent manual and automated tested code. Possibly unstable since it is pre-release. + * `@staging` - Most recent automated tested code and can sometimes contain experimental features. Is pulled from dev stable code. + * `@dev` - Very unstable and contains experimental features. Automated testing may not show breaks since CI is also updated based on code in dev. + ### Inputs * `packages` - Space delimited list of packages to install. * `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed. +* `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. ### Outputs @@ -74,6 +86,36 @@ jobs: version: 1.0 ``` -## Cache Limits +## Caveats + +### Non-file Dependencies + +This action is based on the principle that most packages can be cached as a fileset. There are situations though where this is not enough. + +* Pre and post installation scripts needs to be ran from `/var/lib/dpkg/info/{package name}.[preinst, postinst]`. +* The Debian package database needs to be queried for scripts above (i.e. `dpkg-query`). + +The `execute_install_scripts` argument can be used to attempt to execute the install scripts but they are no guaranteed to resolve the issue. + +```yaml +- uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: mypackage + version: 1.0 + execute_install_scripts: true +``` + +If this does not solve your issue, you will need to run `apt-get install` as a separate step for that particular package unfortunately. + +```yaml +run: apt-get install mypackage +shell: bash +``` + +Please reach out if you have found a workaround for your scenario and it can be generalized. There is only so much this action can do and can't get into the area of reverse engineering Debian package manager. It would be beyond the scope of this action and may result in a lot of extended support and brittleness. Also, it would be better to contribute to Debian packager instead at that point. + +For more context and information see [issue #57](https://github.com/awalsh128/cache-apt-pkgs-action/issues/57#issuecomment-1321024283) which contains the investigation and conclusion. + +### Cache Limits A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted. diff --git a/action.yml b/action.yml index 0f004ba..d3312c8 100644 --- a/action.yml +++ b/action.yml @@ -11,11 +11,17 @@ inputs: required: true default: '' version: - description: 'Version will create a new cache and install packages.' + description: 'Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.' required: false - default: '' + default: '' + execute_install_scripts: + description: 'Execute Debian package pre and post install script upon restore. See README.md caveats for more information.' + required: false + default: 'false' refresh: - description: 'Option to refresh / upgrade the packages in the same cache.' + description: 'OBSOLETE, use version instead.' + debug: + description: 'Enable debugging when there are issues with action. Minor performance penalty.' required: false default: 'false' @@ -40,6 +46,8 @@ runs: ${{ github.action_path }}/pre_cache_action.sh \ ~/cache-apt-pkgs \ "${{ inputs.version }}" \ + "${{ inputs.execute_install_scripts }}" \ + "${{ inputs.debug }}" \ ${{ inputs.packages }} echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV shell: bash @@ -56,8 +64,17 @@ runs: ~/cache-apt-pkgs \ / \ "${{ steps.load-cache.outputs.cache-hit }}" \ + "${{ inputs.execute_install_scripts }}" \ + "${{ inputs.debug }}" \ ${{ inputs.packages }} function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; }; echo "package-version-list=$(create_list main)" >> $GITHUB_OUTPUT echo "all-package-version-list=$(create_list all)" >> $GITHUB_OUTPUT shell: bash + + - id: upload-logs + if: ${{ inputs.debug }} == "true" + uses: actions/upload-artifact@v3 + with: + name: cache-apt-pkgs-logs%${{ inputs.packages }}%${{ inputs.version }} + path: ~/cache-apt-pkgs/*.log diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 060194c..f94b87c 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -3,6 +3,11 @@ # Fail on any error. set -e +# Debug mode for diagnosing issues. +# Setup first before other operations. +debug="${2}" +test ${debug} == "true" && set -x + # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" @@ -11,7 +16,7 @@ source "${script_dir}/lib.sh" cache_dir="${1}" # List of the packages to use. -input_packages="${@:2}" +input_packages="${@:3}" # Trim commas, excess spaces, and sort. normalized_packages="$(normalize_package_list "${input_packages}")" @@ -40,8 +45,13 @@ log "done" log_empty_line log "Updating APT package list..." -sudo apt-fast update > /dev/null -log "done" +last_update_delta_s=$(($(date +%s) - $(date +%s -r /var/cache/apt/pkgcache.bin))) +if test $last_update_delta_s -gt 300; then + sudo apt-fast update > /dev/null + log "done" +else + log "skipped (fresh by ${last_update_delta_s} seconds)" +fi log_empty_line @@ -71,18 +81,20 @@ log_empty_line installed_package_count=$(wc -w <<< "${installed_packages}") log "Caching ${installed_package_count} installed packages..." for installed_package in ${installed_packages}; do - cache_filepath="${cache_dir}/${installed_package}.tar.gz" + cache_filepath="${cache_dir}/${installed_package}.tar" # Sanity test in case APT enumerates duplicates. if test ! -f "${cache_filepath}"; then read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") log " * Caching ${installed_package_name} to ${cache_filepath}..." - # Pipe all package files (no folders) to Tar. - dpkg -L "${installed_package_name}" | - while IFS= read -r f; do - if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows - done | - xargs tar -czf "${cache_filepath}" -C / + + # Pipe all package files (no folders) and installation control data to Tar. + { dpkg -L "${installed_package_name}" \ + & get_install_filepath "" "${package_name}" "preinst" \ + & get_install_filepath "" "${package_name}" "postinst"; } | + while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | + sudo xargs tar -cf "${cache_filepath}" -C / + log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))." fi diff --git a/lib.sh b/lib.sh index 0aedaee..c2f1ec9 100755 --- a/lib.sh +++ b/lib.sh @@ -1,20 +1,40 @@ #!/bin/bash -# Sort these packages by name and split on commas. -function normalize_package_list { - local stripped=$(echo "${1}" | sed 's/,//g') - # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - echo "${sorted}" +############################################################################### +# Execute the Debian install script. +# Arguments: +# Root directory to search from. +# File path to cached package archive. +# Installation script extension (preinst, postinst). +# Parameter to pass to the installation script. +# Returns: +# Filepath of the install script, otherwise an empty string. +############################################################################### +function execute_install_script { + local package_name=$(basename ${2} | awk -F\: '{print $1}') + local install_script_filepath=$(\ + get_install_filepath "${1}" "${package_name}" "${3}") + if test ! -z "${install_script_filepath}"; then + log "- Executing ${install_script_filepath}..." + # Don't abort on errors; dpkg-trigger will error normally since it is + # outside its run environment. + sudo sh -x ${install_script_filepath} ${4} || true + log " done" + fi } -# Gets a list of installed packages as space delimited pairs with each pair colon delimited. +############################################################################### +# Gets a list of installed packages from a Debian package installation log. +# Arguments: +# The filepath of the Debian install log. +# Returns: +# The list of space delimited pairs with each pair colon delimited. # : ... +############################################################################### function get_installed_packages { - install_log_filepath="${1}" + local install_log_filepath="${1}" local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" - dep_packages="" + local dep_packages="" while read -r line; do if [[ "${line}" =~ ${regex} ]]; then dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " @@ -30,7 +50,13 @@ function get_installed_packages { fi } -# Split fully qualified package into name and version. +############################################################################### +# Splits a fully qualified package into name and version. +# Arguments: +# The colon delimited package pair or just the package name. +# Returns: +# The package name and version pair. +############################################################################### function get_package_name_ver { IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. @@ -40,17 +66,107 @@ function get_package_name_ver { echo "${name}" "${ver}" } +############################################################################### +# Gets the package name from the cached package filepath in the +# path/to/cache/dir/:.tar format. +# Arguments: +# Filepath to the cached packaged. +# Returns: +# The package name. +############################################################################### +function get_package_name_from_cached_filepath { + basename ${cached_pkg_filepath} | awk -F\: '{print $1}' +} + +############################################################################### +# Gets the Debian install script file location. +# Arguments: +# Root directory to search from. +# Name of the unqualified package to search for. +# Extension of the installation script (preinst, postinst) +# Returns: +# Filepath of the script file, otherwise an empty string. +############################################################################### +function get_install_filepath { + # Filename includes arch (e.g. amd64). + local filepath="$(\ + ls -1 ${1}var/lib/dpkg/info/${2}*.${3} 2> /dev/null \ + | grep -E ${2}'(:.*)?.'${3} | head -1 || true)" + test "${filepath}" && echo "${filepath}" +} + +############################################################################### +# Gets the relative filepath acceptable by Tar. Just removes the leading slash +# that Tar disallows. +# Arguments: +# Absolute filepath to archive. +# Returns: +# The relative filepath to archive. +############################################################################### +function get_tar_relpath { + local filepath=${1} + if test ${filepath:0:1} = "/"; then + echo "${filepath:1}" + else + echo "${filepath}" + fi +} + function log { echo "$(date +%H:%M:%S)" "${@}"; } function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } -# Writes the manifest to a specified file. -function write_manifest { - log "Writing ${1} packages manifest to ${3}..." - # 0:-1 to remove trailing comma, delimit by newline and sort. - echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} - log "done" +############################################################################### +# Sorts given packages by name and split on commas. +# Arguments: +# The comma delimited list of packages. +# Returns: +# Sorted list of space delimited packages. +############################################################################### +function normalize_package_list { + local stripped=$(echo "${1}" | sed 's/,//g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(\ + echo "${stripped}" \ + | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" + echo "${sorted}" } -get_installed_packages "/tmp/cache-apt-pkgs-action-cache/install.log" +############################################################################### +# Validates an argument to be of a boolean value. +# Arguments: +# Argument to validate. +# Variable name of the argument. +# Exit code if validation fails. +# Returns: +# Sorted list of space delimited packages. +############################################################################### +function validate_bool { + if test "${1}" != "true" -a "${1}" != "false"; then + log "aborted" + log "${2} value '${1}' must be either true or false (case sensitive)." + exit ${3} + fi +} + +############################################################################### +# Writes the manifest to a specified file. +# Arguments: +# Type of manifest being written. +# List of packages being written to the file. +# File path of the manifest being written. +# Returns: +# Log lines from write. +############################################################################### +function write_manifest { + if [ ${#2} -eq 0 ]; then + log "Skipped ${1} manifest write. No packages to install." + else + log "Writing ${1} packages manifest to ${3}..." + # 0:-1 to remove trailing comma, delimit by newline and sort. + echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} + log "done" + fi +} diff --git a/post_cache_action.sh b/post_cache_action.sh index eaae06d..25e7da3 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -12,20 +12,26 @@ cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. +# WARNING: If non-root, this can cause errors during install script execution. cache_restore_root="${2}" # Indicates that the cache was found. cache_hit="${3}" -# List of the packages to use. -packages="${@:4}" +# Cache and execute post install scripts on restore. +execute_install_scripts="${4}" -script_dir="$(dirname -- "$(realpath -- "${0}")")" +# Debug mode for diagnosing issues. +debug="${5}" +test ${debug} == "true" && set -x + +# List of the packages to use. +packages="${@:6}" if [ "$cache_hit" == true ]; then - ${script_dir}/restore_pkgs.sh ~/cache-apt-pkgs "${cache_restore_root}" + ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else - ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages} + ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" ${packages} fi log_empty_line diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 82ba1fe..4b3ff4b 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -1,5 +1,11 @@ #!/bin/bash +# Debug mode for diagnosing issues. +# Setup first before other operations. +debug="${4}" +validate_bool "${debug}" debug 1 +test ${debug} == "true" && set -x + # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" @@ -10,8 +16,14 @@ cache_dir="${1}" # Version of the cache to create or load. version="${2}" +# Execute post-installation script. +execute_install_scripts="${3}" + +# Debug mode for diagnosing issues. +debug="${4}" + # List of the packages to use. -input_packages="${@:3}" +input_packages="${@:5}" # Trim commas, excess spaces, and sort. packages="$(normalize_package_list "${input_packages}")" @@ -23,16 +35,18 @@ log "Validating action arguments (version='${version}', packages='${packages}'). if grep -q " " <<< "${version}"; then log "aborted" log "Version value '${version}' cannot contain spaces." >&2 - exit 1 + exit 2 fi # Is length of string zero? if test -z "${packages}"; then log "aborted" log "Packages argument cannot be empty." >&2 - exit 2 + exit 3 fi +validate_bool "${execute_install_scripts}" execute_install_scripts 4 + log "done" log_empty_line @@ -43,7 +57,7 @@ for package in ${packages}; do if test ! "$(apt-cache show "${package}")"; then echo "aborted" log "Package '${package}' not found." >&2 - exit 3 + exit 5 fi read package_name package_ver < <(get_package_name_ver "${package}") versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index d58c07c..fec5b29 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -3,6 +3,11 @@ # Fail on any error. set -e +# Debug mode for diagnosing issues. +# Setup first before other operations. +debug="${4}" +test ${debug} == "true" && set -x + # Include library. script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" @@ -13,6 +18,10 @@ cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. cache_restore_root="${2}" +test -d ${cache_restore_root} || mkdir ${cache_restore_root} + +# Cache and execute post install scripts on restore. +execute_install_scripts="${3}" cache_filepaths="$(ls -1 "${cache_dir}" | sort)" log "Found $(echo ${cache_filepaths} | wc -w) files in the cache." @@ -31,12 +40,22 @@ log "done" log_empty_line # Only search for archived results. Manifest and cache key also live here. -cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) +cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar | sort) cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w) + log "Restoring ${cached_pkg_filecount} packages from cache..." for cached_pkg_filepath in ${cached_pkg_filepaths}; do + log "- $(basename "${cached_pkg_filepath}") restoring..." sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null log " done" + + # Execute install scripts if available. + if test ${execute_install_scripts} == "true"; then + # May have to add more handling for extracting pre-install script before extracting all files. + # Keeping it simple for now. + execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install + execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure + fi done log "done"