cranky

#!/bin/bash

function send_to_log {
    echo $(date +"%R:%S %Z %:z %Y-%m-%d:") "$(basename $0): [$1] $2" >&2
}

function log_error {
    send_to_log "ERROR" "$1"
}

function log_warning {
    send_to_log "WARNING" "$1"
}

function log_info {
    send_to_log "INFO" "$1"
}

Theory=("sft.cern.ch" "grid.cern.ch" "cernvm-prod.cern.ch" "alice.cern.ch")

function is_uint {
    # test if $1 is unsigned integer
    case $1 in
      '' | *[!0-9]*)
        return 1
        ;;
    esac
}


function version_is_lower {
    # expects the classical version pattern major.minor.release[.subrelease...]
    # with major, minor, release being unsigned integers
    # $1: version to be tested
    # $2: pattern to be tested against
    # $3: controls the #sections to be compared (from left to right)
    # returns
    # 0: in case of $1 is lower than $2
    # 1: in case of $1 is equal or higher than $2
    # 2: in case of an error
    
    is_uint "$3" || return 2
    sections=$3
    
    IFS=. read -a ver_arr <<<"$1" || return 2
    IFS=. read -a pat_arr <<<"$2" || return 2
    
    (( ${#ver_arr[@]} < $sections )) || (( ${#pat_arr[@]} < $sections )) && return 2

    for (( i=0; i<$sections; i++ )); do
        is_uint "${ver_arr[$i]}" || return 2
        is_uint "${pat_arr[$i]}" || return 2
    done
    
    for (( i=0; i<$sections; i++ )); do
        (( ${ver_arr[$i]} < ${pat_arr[$i]} )) && return 0
        (( ${ver_arr[$i]} > ${pat_arr[$i]} )) && return 1
    done
    
    ((sections--))
    (( ${ver_arr[$sections]} == ${pat_arr[$sections]} )) && return 1
    return 2
}


function get_boinc_info {
    if [ ! -f init_data.xml ] ; then
	log_error "'init_data.xml' is missing."
        return 1
    fi
    attribute=$(grep "^<$1>" init_data.xml | cut -d '>' -f2 | cut -d '<' -f1)
    echo ${attribute}
}

function check_cvmfs_legacy {
    log_info "Checking CVMFS."
    cvmfs_config=$(which cvmfs_config 2>/dev/null)
    if [ $? -gt 0 ] ; then
        log_error "'which' could not locate the command 'cvmfs_config'."
        return 1
    fi

    if [ ! -d /cvmfs ] ; then
        log_error "CVMFS mount point /cvmfs not found."
        return 1
    fi

    repos=$app[@]
    for repo in ${!repos}; do
	for command in probe stat ; do
            cvmfs_config ${command} ${repo}
	    if [ $? -gt 0 ] ; then
                log_error "'cvmfs_config ${command} ${repo}' failed."
                return 1
	    fi
	done
    done
}

function check_runc_legacy {
    log_info "Checking runc."
    if grep -q cgroup2 /proc/mounts; then
        runc="/cvmfs/grid.cern.ch/vc/containers/runc.new"
    else
	runc="/cvmfs/grid.cern.ch/vc/containers/runc.old"
    fi
    if [ ! -e ${runc} ] ; then
        log_error "${runc} does not exist."
        return 1
    fi
    ${runc} -v > /dev/null
    if [ $? -gt 0 ] ; then
        log_error "'runc -v' failed."
        return 1
    fi
    if [ -e /proc/sys/user/max_user_namespaces ] ; then
        value=$(cat /proc/sys/user/max_user_namespaces)
        if [ ${value} -eq 0 ] ; then
            log_error "max_user_namespaces in /proc/sys/user/ is set to 0."
            return 1
	fi
    fi
    return
}

function create_filesystem {
    log_info "Creating container filesystem."
    rm -frd cernvm
    mkdir -p cernvm/rootfs
    root="/cvmfs/cernvm-prod.cern.ch/cvm3"
    log_info "Using ${root}"
    paths=('/srv' '/tmp' '/etc' '/usr' '/usr/local' '/var' '/var/cache' '/var/cvs' '/var/db' '/var/empty'
           '/var/lib' '/var/local' '/var/lock' '/var/log' '/var/run' '/var/tmp' '/root' '/home' '/var/spool')
    for path in ${paths[*]} ; do
        mkdir cernvm/rootfs${path}
    done

    paths=('/bin' '/etc' '/lib' '/lib64' '/opt' '/sbin' '/usr/bin' '/usr/doc' '/usr/etc' '/usr/include'
	   '/usr/lib' '/usr/lib64' '/usr/libexec' '/usr/sbin' '/usr/share' '/usr/src' '/usr/vice')
    for path in ${paths[*]} ; do
        ln -sf ${root}${path} cernvm/rootfs${path}
    done
    cp /cvmfs/grid.cern.ch/vc/containers/cernvm/config.json cernvm/config.json
    mkdir -p cernvm/shared/html/job

}

function update_config_legacy {
#     log_info "Updating config.json."

    sed -i "s/\"hostID\": 122/\"hostID\": $(id -u)/" cernvm/config.json
    sed -i "s/\"hostID\": 129/\"hostID\": $(id -g)/" cernvm/config.json
    slot=$(basename $(pwd))
    if [ -d /sys/fs/cgroup/freezer/boinc/${slot} ]; then
	sed -i "s/\"linux\": {/\"linux\": {\n \t\"cgroupsPath\": \"\/boinc\/${slot}\",/" cernvm/config.json
    fi
}

function update_config {
#    log_info "Updating config.json."

    sed -e "s/\"hostID\": 122/\"hostID\": $(id -u)/" \
        -e "s/\"hostID\": 129/\"hostID\": $(id -g)/" \
        -i cernvm/config.json
}

function prepare_input {
    cp init_data.xml cernvm/shared/init_data.xml
    mkdir cernvm/shared/bin
    cp /cvmfs/grid.cern.ch/vc/containers/cernvm/copilot-config cernvm/shared/bin/copilot-config
    chmod 755 cernvm/shared/bin/copilot-config
    cp input cernvm/shared/job
    chmod 755 cernvm/shared/job
    sed -i 's/tar xzm/tar xzmo/' cernvm/shared/job
    if [ $? -eq 1 ] ; then
        log_error "'sed of input to add no-save-owner."
        return 1
    fi
}

function prepare_output {
    log_info "Preparing output."
    if [ -f cernvm/shared/runRivet.log ]; then
	mkdir -p shared
	tar -zcf shared/output.tgz  --exclude bin --exclude runPost.sh  --exclude html --exclude init_data.xml -C cernvm/shared .
	ln -sf shared/output.tgz output.tgz  # To be compatible with the VBox App\
	return 0
    else
        log_error "No output found."
	return 1
    fi
}

function pause_legacy {
    result_id=$(get_boinc_info result_name)
    log_info "Pausing container ${result_id}."
    if grep -q cgroup2 /proc/mounts; then
        runc="/cvmfs/grid.cern.ch/vc/containers/runc.new"
    else
	runc="/cvmfs/grid.cern.ch/vc/containers/runc.old"
    fi
    if [ -f /sys/fs/cgroup/freezer/boinc/freezer.state || -d /sys/fs/cgroup/freezer/boinc ]; then
	${runc} --root state pause ${result_id}
    else
	log_warning "Cannot pause container as /sys/fs/cgroup/freezer/boinc/freezer.state or /sys/fs/cgroup/freezer/boinc do not exist."
    fi
}

function pause {
    result_id=$(get_boinc_info result_name)
    log_info "Pausing systemd unit ${result_id}.scope"
    sudo -n /usr/bin/systemctl freeze ${result_id}.scope
}

function resume_legacy {
    result_id=$(get_boinc_info result_name)
    log_info "Resuming container ${result_id}."
    /cvmfs/grid.cern.ch/vc/containers/runc --root state resume ${result_id}
}

function resume {
    result_id=$(get_boinc_info result_name)
    log_info "Resuming systemd unit ${result_id}.scope"
    sudo -n /usr/bin/systemctl thaw ${result_id}.scope
}

function create_cgroup_legacy {
    if [ -d /sys/fs/cgroup/freezer/boinc ]; then
        slot=$(basename $(pwd))
        log_info "Creating cgroup for slot ${slot}"
        CGROUPS=( freezer cpuset devices memory "cpu,cpuacct" pids blkio hugetlb net_cls net_prio perf_event freezer )
        CGROUP_MOUNT="/sys/fs/cgroup"
        CGROUP_PATH="boinc/${slot}"
        for cg in "${CGROUPS[@]}"
        do
	    mkdir -p "$CGROUP_MOUNT/$cg/$CGROUP_PATH"
        done
    else
        CGROUPS=( cpuset memory io pids freezer )
        CGROUP_MOUNT="/sys/fs/cgroup/unified"
        CGROUP_PATH="boinc"

        for cg in "${CGROUPS[@]}"
        do
            mkdir -p "$CGROUP_MOUNT/$cg/$CGROUP_PATH"
        done
    fi
}

function run_legacy {
    log_info "Running Container 'runc'."
    if grep -q cgroup2 /proc/mounts; then
        runc="/cvmfs/grid.cern.ch/vc/containers/runc.new"
    else
	runc="/cvmfs/grid.cern.ch/vc/containers/runc.old"
    fi
    result_id=$(get_boinc_info result_name)
    trap pause_legacy SIGTSTP
    trap resume_legacy SIGCONT
    ${runc} --root state run -b cernvm ${result_id} &
    child=$!
    log_info "mcplots runspec: $(grep -Poam1 'runspec=\Kboinc[^"]*' input 2>/dev/null)"
    while true
    do
	wait ${child}
	status=$?
	if [ ! ${status} -eq 146 ] && [ ! ${status} -eq 148 ]; then
	    log_info "Container 'runc' finished with status code ${status}."
	    return
	fi
    done
}

function run {
    result_id=$(get_boinc_info result_name)
    log_info "Starting runc container."
    log_info "To get some details on systemd level run"
    log_info "systemctl status ${result_id}.scope"
    log_info "mcplots runspec: $(grep -Poam1 'runspec=\Kboinc[^"]*' input 2>/dev/null)"
    log_info "----,^^^^,<<<~_____---,^^^,<<~____--,^^,<~__;_"

    run_uid="$(id -un 2>/dev/null)"
    run_gid="boinc"
    run_BindsTo="$(ps --no-headers -o unit $$ 2>/dev/null)"
    run_After="${run_BindsTo}"

    trap resume SIGCONT
    trap pause SIGTSTP
    sudo -n /usr/bin/systemd-run --scope -u ${result_id} -p BindsTo=${run_BindsTo} -p After=${run_After} --slice-inherit --uid=${run_uid} --gid=${run_gid} --same-dir -q -G ${runc_bin} --root state run -b cernvm ${result_id} &

    child=$!

    while :; do
        wait ${child}
        status=$?
        if [[ $status != 146 ]] && [[ $status != 148 ]]; then
            trap - SIGTSTP SIGCONT
            log_info "Container $result_id finished with status code ${status}."
            return
        fi
    done
}


function fail {
    exit 206
}

function early_fail {
    log_info "Early shutdown initiated due to previous errors."
    log_info "Cleanup will take a few minutes..."

    # a modern multi CPU computer often starts many tasks within a very short period
    # if they all fail due to missing requirements delay shutdown
    sleep $(shuf -n 1 -i 720-900)

    exit 206
}


function check_cvmfs_configuration_offline {
    export required_repos=("alice.cern.ch" "cernvm-prod.cern.ch" "grid.cern.ch" "sft.cern.ch")
    ret_ccc_off=0

    cvmfs_config_file="/etc/cvmfs/default.local"
    if [[ ! -e "$cvmfs_config_file" ]]; then
        # must exist since this is the file that contains the repo list
        log_info "Can't find '$cvmfs_config_file'."
        log_error "This file is expected to contain the CVMFS repository list."
        (( ret_ccc_off < 2 )) && ret_ccc_off=2
    else
        file_content="$(tac $cvmfs_config_file)"
        cvmfs_config_patterns=("CVMFS_USE_CDN=yes"\
                               "CVMFS_HTTP_PROXY=\"auto;DIRECT\""\
                              )
        for pattern in "${cvmfs_config_patterns[@]}"; do
            if ! grep -qm1 "^\s*${pattern}" <<<"$file_content" 2>/dev/null; then
                # minor errors
                # just print a hint what should be set on a normal volunteer's computer
                log_info "Missing '$pattern' in '$cvmfs_config_file'."
                #(( ret_ccc_off < 1 )) && ret_ccc_off=1
            fi
        done

        my_repositories="$(grep -Pom1 "^\s*CVMFS_REPOSITORIES=[\"\']?\K[ a-zA-Z0-9,._-]+[^\"\']" <<<"$file_content" 2>/dev/null)"
        if [[ -z "$my_repositories" ]]; then
            log_info "No repositories configured in '$cvmfs_config_file'."
            (( ret_ccc_off < 2 )) && ret_ccc_off=2
        else
            IFS=", " read -a conf_repo_list <<<"$my_repositories"
            for req_repo in "${required_repos[@]}"; do
                for conf_repo in "${conf_repo_list[@]}"; do
                    if [[ "$req_repo" == "$conf_repo" ]] || [[ "${req_repo%.cern.ch}" == "$conf_repo" ]]; then
                        break 2
                    fi
                done
                log_info "Missing required Repository '$req_repo' in '$cvmfs_config_file'."
                (( ret_ccc_off < 2 )) && ret_ccc_off=2
            done
        fi
    fi

    cvmfs_config_file="/etc/cvmfs/domain.d/cern.ch.local"
    if [[ ! -e "$cvmfs_config_file" ]]; then
        # minor error
        # just print a hint what should be set on a normal volunteer's computer
        log_info "Can't find '$cvmfs_config_file'."
        #(( ret_ccc_off < 1 )) && ret_ccc_off=1
    else
        file_content="$(tac $cvmfs_config_file)"
        cvmfs_config_patterns=("CVMFS_CONFIG_REPO_REQUIRED=yes")
        for pattern in "${cvmfs_config_patterns[@]}"; do
            if ! grep -qm1 "^\s*${pattern}" <<<"$file_content" 2>/dev/null; then
                # minor error
                # just print a hint what should be set on a normal volunteer's computer
                log_info "Missing '$pattern' in '$cvmfs_config_file'."
                #(( ret_ccc_off < 1 )) && ret_ccc_off=1
            fi
        done
    fi

    cvmfs_config_file="/etc/cvmfs/config.d/cvmfs-config.cern.ch.local"
    if [[ ! -e "$cvmfs_config_file" ]]; then
        # minor error
        # just print a hint what should be set on a normal volunteer's computer
        log_info "Can't find '$cvmfs_config_file'."
        #(( ret_ccc_off < 1 )) && ret_ccc_off=1
    else
        file_content="$(tac $cvmfs_config_file)"
        cvmfs_config_patterns=("CVMFS_CONFIG_REPO_REQUIRED=no")
        for pattern in "${cvmfs_config_patterns[@]}"; do
            if ! grep -qm1 "^\s*${pattern}" <<<"$file_content" 2>/dev/null; then
                # minor error
                # just print a hint what should be set on a normal volunteer's computer
                log_info "Missing '$pattern' in '$cvmfs_config_file'."
                #(( ret_ccc_off < 1 )) && ret_ccc_off=1
            fi
        done
    fi

    return $ret_ccc_off
}


function probe_cvmfs_repos {
    cvmfs_config probe "$1" >"${my_tmp_dir}/probe_$1" 2>/dev/null
    echo $? >"${my_tmp_dir}/result_probe_$1"
}
# must be exported to be available for child shells
export -f probe_cvmfs_repos


function check_cvmfs_configuration_online {
    ret_ccc_onl=0

    my_umask="$(umask)"
    umask 077
    export my_tmp_dir="$(mktemp -dp $(pwd))"
    umask "${my_umask}"

    bg_pids_probe_arr=()
    for cv_repo in "${required_repos[@]}"; do
        probe_cvmfs_repos "${cv_repo}" &
        bg_pids_probe_arr+=($!)
    done

    for bg_pid in ${bg_pids_probe_arr[*]}; do
        wait $bg_pid
    done

    for cv_repo in "${required_repos[@]}"; do
        log_info "$(cat "${my_tmp_dir}/probe_$cv_repo")"
    done

    if grep -v '^0$' <(cat "${my_tmp_dir}/result_probe_"* 2>&1) >/dev/null 2>&1; then
        (( ret_ccc_onl < 2 )) && ret_ccc_onl=2
    fi

    rm -frd "${my_tmp_dir}" &

    if (( ret_ccc_onl < 2 )); then
        # prints a hint to the logfile whether openhtc.io and/or a local proxy is used.
        cvmfs_excerpt=($(cut -d ' ' -f 1,17,18 < <(tail -n1 < <(cvmfs_config stat ${required_repos[0]}))))
        log_info "Excerpt from \"cvmfs_config stat\": VERSION HOST PROXY"
        log_info "$(echo "${cvmfs_excerpt[0]} ${cvmfs_excerpt[1]%"/cvmfs/sft.cern.ch"} ${cvmfs_excerpt[2]}")"
    fi

    return $ret_ccc_onl
}


function check_local_requirements {
    log_info "This application must have permanent access to"
    log_info "online repositories via a local CVMFS service."
    log_info "It supports suspend/resume if a couple of"
    log_info "requirements are fulfilled."
    log_info "Most important:"
    log_info "- init process is systemd"
    log_info "- cgroups v2 is enabled and 'freezer' is available"
    log_info "- the user running this application is a member of the 'boinc' group"
    log_info "- sudo is at least version 1.9.10"
    log_info "- sudoer file provided by LHC@home is installed"
    
    log_info "Checking local requirements."

    # minor issues return "1" but legacy mode may be possible
    # show stoppers return "2"
    ret_clr=0

    # check if init is systemd
    # if not, this system may be very old, run under WSL2 or otherwise exotic
    if ! grep -iqam1 'systemd' /sbin/init 2>/dev/null; then
        log_info "Init process is not 'systemd'."
        (( ret_clr < 1 )) && ret_clr=1
    fi

    if ! command -v systemd-run >/dev/null 2>&1; then
        log_info "Can't find 'systemd-run'."
        (( ret_clr < 1 )) && ret_clr=1
    fi

    # check if cgroups v2 are available
    if ! ls /sys/fs/cgroup/cgroup.controllers >/dev/null 2>&1; then
        log_info "Can't find 'cgroups v2'."
        (( ret_clr < 1 )) && ret_clr=1
    fi

    # if the user is not a member of the boinc group sudo commands will fail.
    my_user_name="$(id -un)"
    group_name="boinc"
    if ! grep -q "\b${group_name}\b" <(id -Gn $my_user_name); then
        log_info "User '$my_user_name' is missing in group '$group_name'."
        log_info "To add '$my_user_name' to '$group_name' run 'sudo usermod -aG $group_name $my_user_name'."
        (( ret_clr < 2 )) && ret_clr=2
    fi

    sudo_min_version="1.9.10"
    sudo_version="$(grep -Piom1 'version[^0-9]+\K.*' <(sudo -V 2>/dev/null) 2>/dev/null)"
    log_info "Found Sudo-Version $sudo_version."
    
    version_is_lower "${sudo_version%%[^0-9\.]*}" "$sudo_min_version" 3 >/dev/null 2>&1
    case $? in
      0)
        log_info "To run this task in new mode"
        log_info "Sudo-Version must be at least $sudo_min_version."
        (( ret_clr < 1 )) && ret_clr=1
        ;;
      1)
        sudoers_file="/etc/sudoers.d/50-lhcathome_boinc_theory_native"
        if ! grep -qm1 "$sudoers_file" <(sudo -nl 2>/dev/null) 2>/dev/null; then
            log_info "Can't find or read '$sudoers_file'."
            (( ret_clr < 1 )) && ret_clr=1
        else
            if ! sudo -n /usr/bin/cat $sudoers_file >/dev/null 2>&1; then
                log_info "Found '$sudoers_file'"
                log_info "but sudo may not be able to interpret it."
                (( ret_clr < 1 )) && ret_clr=1
            fi
        fi
        ;;
      *)
        log_info "Can't compare sudo version number."
        (( ret_clr < 1 )) && ret_clr=1
        ;;
    esac

    if ! command -v cvmfs_config >/dev/null 2>&1; then
        log_error "Can't find 'cvmfs_config'."
        log_error "This usually means a local CVMFS client is not installed"
        log_error "although it is a MUST to get data from online repositories."
        (( ret_clr < 2 )) && ret_clr=2
    else
        check_cvmfs_configuration_offline
        ret=$?
        (( ret > ret_clr )) && ret_clr=$ret

        if (( ret < 2 )); then
            check_cvmfs_configuration_online
            ret=$?
            (( ret > ret_clr )) && ret_clr=$ret
        fi
    fi

    # recommendation from the old Singularity manual
    # strictly required on older CentOS systems since it
    # is enabled by default on most other Linux distros
    # may only be reported instead of marked as error
    max_uns_path="/proc/sys/user/max_user_namespaces"
    if [[ ! -e "$max_uns_path" ]] || (( $(cat "$max_uns_path" 2>/dev/null) < 15000 )); then
        log_info "Missing 'max_user_namespaces'"
        log_info "or it's value is lower than 15000."
        log_info "Run the following commands as root:"
        log_info "echo \"user.max_user_namespaces = 15000\" > /etc/sysctl.d/90-max_user_namespaces.conf"
        log_info "sysctl -p /etc/sysctl.d/90-max_user_namespaces.conf"
        #(( ret_clr < 2 )) && ret_clr=2
    fi

    runc_bin="$(command -v runc)"
    if  [[ -n "$runc_bin" ]]; then
        runc_version="$(head -n1 <($runc_bin -v))"
        log_info "Found a local $runc_version."
    else
        runc_bin="/cvmfs/grid.cern.ch/vc/containers/runc.new"
        runc_version="$(head -n1 <($runc_bin -v 2>/dev/null))"
        if [[ -n "$runc_version" ]]; then
            log_info "Found '$runc_version' at '$runc_bin'."
        else
            # can we retire this version?
            runc_bin="/cvmfs/grid.cern.ch/vc/containers/runc.old"
            runc_version="$(head -n1 <($runc_bin -v 2>/dev/null))"
            if [[ -n "$runc_version" ]]; then
                log_info "Found '$runc_version' at '$runc_bin'."
            else
                log_info "Can't find 'runc'."
                (( ret_clr < 2 )) && ret_clr=2
            fi
        fi
    fi


    return $ret_clr
}


app=$(get_boinc_info app_name)
 if [ $? -gt 0 ] ; then
     fail
 fi
log_info "Detected ${app} App"
check_local_requirements
ret_code_main=$?
if (( ret_code_main > 1 )); then
    log_error "Major requirements are missing. Can't run this task."
    early_fail
fi

if (( ret_code_main > 0 )); then
    log_info "Minor requirements are missing. Will try to run this task in legacy mode."

    # already checked
    #check_cvmfs_legacy ${app} || early_fail
    check_runc_legacy || early_fail
    create_filesystem
    create_cgroup_legacy
    update_config_legacy
    prepare_input
    run_legacy
    prepare_output || fail
else
    create_filesystem
    update_config
    prepare_input
    run
    prepare_output || fail
fi

rm -rfd cernvm # Clean up