Skip to content

Commit

Permalink
🔨 Scripted build/archive multiple envs
Browse files Browse the repository at this point in the history
  • Loading branch information
@thinkyhead
thinkyhead committed Dec 16, 2024
1 parent dcc1056 commit d5dfd18
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 54 deletions.
9 changes: 8 additions & 1 deletion buildroot/bin/build_all_examples
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# [-f|--nofail] - Don't stop on a failed build
# [-h|--help] - Print usage and exit
# [-l|--limit=#] - Limit the number of builds in this run
# [-m|--many] - Build all the environments for each example
# [-n|--nobuild] - Don't actually build anything
# [-o|--output] - Redirect export / archiving to another location
# (By default export to origin config folders)
Expand Down Expand Up @@ -38,6 +39,7 @@ build_all_examples [-a|--archive] - Copy the binary to the export locati
[-f|--nofail] - Don't stop on a failed build
[-h|--help] - Print usage and exit
[-l|--limit=#] - Limit the number of builds in this run
[-m|--many] - Build all the environments for each example
[-n|--nobuild] - Don't actually build anything
[-o|--output] - Redirect export / archiving to another location
(By default export to origin config folders)
Expand All @@ -53,7 +55,7 @@ unset FIRST_CONF
EXIT_USAGE=
LIMIT=1000

while getopts 'aB:b:cde:fhl:no:pr:sv-:' OFLAG; do
while getopts 'aB:b:cde:fhl:mno:pr:sv-:' OFLAG; do
case "${OFLAG}" in
a) ARCHIVE=1 ; bugout "Archiving" ;;
B) CBASE=${OPTARG%/} ; bugout "Base: $CBASE" ;;
Expand All @@ -64,6 +66,7 @@ while getopts 'aB:b:cde:fhl:no:pr:sv-:' OFLAG; do
f) NOFAIL=1 ; bugout "Continue on Fail" ;;
h) EXIT_USAGE=1 ; break ;;
l) LIMIT=$OPTARG ; bugout "Limit to $LIMIT build(s)" ;;
m) MANY=1 ; bugout "Many Envs" ;;
n) DRYRUN=1 ; bugout "Dry Run" ;;
o) OUTBASE="${OPTARG%/}" ; bugout "Archive to $OUTBASE" ;;
p) PURGE=1 ; bugout "Purge stat file" ;;
Expand All @@ -74,6 +77,7 @@ while getopts 'aB:b:cde:fhl:no:pr:sv-:' OFLAG; do
archive) ARCHIVE=1 ; bugout "Archiving" ;;
base) CBASE=${OVAL%/} ; bugout "Base: $CBASE" ;;
branch) BRANCH=$OVAL ; bugout "Branch: $BRANCH" ;;
many) MANY=1 ; bugout "Many Envs" ;;
nofail) NOFAIL=1 ; bugout "Continue on Fail" ;;
resume) ISRES=1 ; FIRST_CONF=$OVAL ; bugout "Resume: $FIRST_CONF" ;;
continue) CONTINUE=1 ; bugout "Continue" ;;
Expand Down Expand Up @@ -179,6 +183,9 @@ find -ds "$CBASE"/config/examples -type d -name 'Configuration.h' -o -name 'Conf
# Exporting? Add -e argument
((CEXPORT)) && CARGS+=("-e" "$CEXPORT")

# Build many environments? Add -m argument
((NOFAIL)) && CARGS+=("-m")

# Continue on fail? Add -f argument
((NOFAIL)) && CARGS+=("-f")

Expand Down
148 changes: 96 additions & 52 deletions buildroot/bin/build_example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# build_example -b|--base=<path> - Configurations root folder (e.g., ./.pio/build-BRANCH)
# -c|--config=<rel> - Sub-path of the configs to build (within config/examples)
# [-n|--index=N] - Which environment to build, by index (Based on pins.h comments)
# [-m|--many] - Build all the board's environments listed in pins.h
# [-e|--export=N] - Use CONFIG_EXPORT N to export the config to the export location
# [-a|--archive] - Archive the build (to the export location)
# [-o|--output] - Redirect export / archiving to another location
Expand All @@ -21,6 +22,7 @@ usage() { echo "Usage:
build_example -b|--base=<path> - Configurations root folder (e.g., ./.pio/build-BRANCH)
-c|--config=<rel> - Sub-path of the configs to build (within config/examples)
[-n|--index=N] - Which environment to build, by index (Based on pins.h comments)
[-m|--many] - Build all the board's environments listed in pins.h
[-e|--export=N] - Use CONFIG_EXPORT N to export the config to the export location
[-a|--archive] - Archive the build (to the export location)
[-o|--output] - Redirect export / archiving to another location
Expand Down Expand Up @@ -53,15 +55,17 @@ EXPNUM=
NOFAIL=
OUTBASE=
BUILDINDEX=1
MANY=

while getopts 'ab:c:e:fhn:o:r-:' OFLAG; do
while getopts 'ab:c:e:fhmn:o:r-:' OFLAG; do
case "${OFLAG}" in
a) ARCHIVE=1 ;;
b) BASE="${OPTARG%/}" ;;
c) CONFIG="${OPTARG%/}" ;;
e) EXPNUM="$OPTARG" ;;
f) NOFAIL=1 ;;
h) EXIT_USAGE=1 ; break ;;
m) MANY=1 ;;
n) BUILDINDEX="$OPTARG" ;;
o) OUTBASE="${OPTARG%/}" ;;
r) REVEAL=1 ;;
Expand All @@ -71,6 +75,7 @@ while getopts 'ab:c:e:fhn:o:r-:' OFLAG; do
allow) ALLOW=1 ;;
base) BASE="${OVAL%/}" ;;
config) CONFIG="${OVAL%/}" ;;
many) MANY=1 ;;
index) BUILDINDEX="$OVAL" ;;
export) EXPNUM="$OVAL" ;;
output) OUTBASE="${OVAL%/}" ;;
Expand Down Expand Up @@ -178,68 +183,107 @@ fi

((ARCHIVE)) && find "$BUILD" -type f \( "${BNAME[@]}" \) -exec rm "{}" \;

set +e
echo "Building example $CONFIG..."

# If doing many builds get a list of all environment names,
# which also gives us the number of environments.
if ((MANY)); then
ENVLIST=$(mfenvs) # BOARD_NAME_STRING (1234): [ env1 env2 env3 ... ]
ENVLIST=${ENVLIST##*: [ }
ENVARRAY=(${ENVLIST% ]})
ENVCOUNT=${#ENVARRAY[*]}
((ENVCOUNT)) || { alrt "mfenvs failed for this board." ; exit 1 ; }
echo "Found $ENVCOUNT environment(s): ${ENVARRAY[*]}"
fi

echo "Building example $CONFIG ..."
mftest -s -a -n$BUILDINDEX ; ERR=$?
# Run one or more builds based on --many
# Build all from BUILDINDEX onward (usually 1) meaning ALL.
# MANY with a BUILDINDEX may be useful for continuing an interrupted build.

((ERR)) && alrt "Failed ($ERR)" || annc "Success"
while ((1)); do
set +e

set -e
echo "Building example $CONFIG ($BUILDINDEX)..."

# Run a build and record the error number
mftest -s -a -n$BUILDINDEX ; ERR=$?

# "Index out of range" can fail without an error
((MANY)) && ((ERR == 66)) && ERR=0 && break # "index out of range"

# Short message reporting Error or Success
((ERR)) && alrt "Failed ($ERR)" || annc "Success"

set -e

if [[ $ERR -gt 0 ]]; then
if [[ $ERR -gt 0 ]]; then

# Error? For --nofail simply log. Otherwise return the error.
if [[ -n $NOFAIL ]]; then
date +"%F %T [FAIL] $CONFIG" >>./.pio/error-log.txt
else
exit $ERR
fi

# Error? For --nofail simply log. Otherwise return the error.
if [[ -n $NOFAIL ]]; then
date +"%F %T [FAIL] $CONFIG" >>./.pio/error-log.txt
else
exit $ERR
fi

else
# Copy exports back to the configs
if [[ -n $EXPNUM ]]; then
annc "Exporting $EXPNUM"
[[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; }
find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \;
fi

# When building many, create sub-folders for each build env name
if [[ -n $MANY && $ENVCOUNT -gt 1 ]]; then
ENV=${ENVARRAY[BUILDINDEX-1]}
ARCENVSUB="$ARCSUB/$ENV"
else
ARCENVSUB="$ARCSUB"
fi

# Copy potential firmware files into the config folder
# TODO: Consider firmware that needs an STM32F4_UPDATE folder.
# Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality
if ((ARCHIVE)); then
annc "Archiving"
find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c '
ARCDIR="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3
NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*}
ZIPX=
if [[ $CONFIG == *Simulator* ]]; then
case $(uname | tr '[:upper:]' '[:lower:]') in
darwin) SUB="macOS" ; ZIPX="-X" ;;
*linux) SUB="Linux" ;;
win*) SUB="Windows" ;;
msys*) SUB="Windows" ;;
cygwin*) SUB="Windows" ;;
mingw*) SUB="Windows" ;;
*) SUB='Unix' ;;
esac
ARCH=$(uname -m | tr '[:lower:]' '[:upper:]')
ARCDIR="$ARCDIR/$SUB-$ARCH"
fi
mkdir -p "$ARCDIR"
rm -f "$ARCDIR"/*.zip "$ARCDIR"/*.sha256.txt
cd "$DIR"
SHASUM=$(sha256sum "$NAME" | cut -d" " -f1)
echo "$CONFIG\n$SHASUM" > "$ARCDIR/$NAME.sha256.txt"
zip $ZIPX "$ARCDIR/$SHRT.zip" "$NAME" && rm "$NAME"
cd - >/dev/null
' sh "$ARCENVSUB" "$CONFIG" {} +
fi

# Reveal the configs after the build, if requested
((REVEAL)) && { annc "Revealing $ARCENVSUB" ; open "$ARCENVSUB" ; }

# Copy exports back to the configs
if [[ -n $EXPNUM ]]; then
annc "Exporting $EXPNUM"
[[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; }
find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \;
fi

# Copy potential firmware files into the config folder
# TODO: Consider firmware that needs an STM32F4_UPDATE folder.
# Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality
if ((ARCHIVE)); then
annc "Archiving"
find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c '
ARCSUB="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3
NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*}
ZIPX=
if [[ $CONFIG == *Simulator* ]]; then
case $(uname | tr '[:upper:]' '[:lower:]') in
darwin) SUB="macOS" ; ZIPX="-X" ;;
*linux) SUB="Linux" ;;
win*) SUB="Windows" ;;
msys*) SUB="Windows" ;;
cygwin*) SUB="Windows" ;;
mingw*) SUB="Windows" ;;
*) SUB='Unix' ;;
esac
ARCH=$(uname -m | tr '[:lower:]' '[:upper:]')
ARCSUB="$ARCSUB/$SUB-$ARCH"
fi
mkdir -p "$ARCSUB"
rm -f "$ARCSUB"/*.zip "$ARCSUB"/*.sha256.txt
cd "$DIR"
SHASUM=$(sha256sum "$NAME" | cut -d" " -f1)
echo "$CONFIG\n$SHASUM" > "$ARCSUB/$NAME.sha256.txt"
zip $ZIPX "$ARCSUB/$SHRT.zip" "$NAME" && rm "$NAME"
cd - >/dev/null
' sh "$ARCSUB" "$CONFIG" {} +
fi
((MANY)) || break # Only one build if not --many

# Reveal the configs after the build, if requested
((REVEAL)) && { annc "Revealing $ARCSUB" ; open "$ARCSUB" ; }
# Set up for the next build, if there is one
((++BUILDINDEX > ENVCOUNT)) && break

fi
done

exit 0
33 changes: 33 additions & 0 deletions buildroot/bin/mfenvs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
#
# mfenvs Print the current board and environment information
# Output -> "SHORT_NAME (###): [ env1 env2 env3 ... ]"
#

[[ -d Marlin/src ]] || { echo "Please 'cd' to the Marlin repo root." ; exit 1 ; }
which pio >/dev/null || { echo "Make sure 'pio' is in your execution PATH." ; exit 1 ; }

errout() { echo -e "\033[0;31m$1\033[0m" ; }

case $(uname | tr '[:upper:]' '[:lower:]') in
darwin) SYS='mac' ;;
*linux) SYS='lin' ;;
win*) SYS='win' ;;
msys*) SYS='win' ;;
cygwin*) SYS='win' ;;
mingw*) SYS='win' ;;
*) SYS='uni' ;;
esac

ACODE='/^[[:space:]]*#define[[:space:]]MOTHERBOARD[[:space:]]/ { sub(/^BOARD_/, "", $3); print $3 }'
MB=$(awk "$ACODE" Marlin/Configuration.h 2>/dev/null)
[[ -z $MB ]] && MB=$(awk "$ACODE" Marlin/Config.h 2>/dev/null)
[[ -z $MB ]] && { echo "Error - Can't read MOTHERBOARD setting." ; exit 1 ; }
BLINE=$( grep -E "define\s+BOARD_$MB\b" Marlin/src/core/boards.h )
BNUM=$( sed -E 's/^.+BOARD_[^ ]+ +([0-9]+).+$/\1/' <<<"$BLINE" )
[[ -z $BNUM ]] && { echo "Error - Can't find BOARD_$MB in core/boards.h." ; exit 1 ; }
ENVS=( $( grep -EA1 "MB\(.*\b$MB\b.*\)" Marlin/src/pins/pins.h | grep -E "#include.+//.+(env|$SYS):[^ ]+" | grep -oE "(env|$SYS):[^ ]+" | sed -E "s/(env|$SYS)://" ) )
[[ -z $ENVS ]] && { errout "Error - Can't find target(s) for $MB ($BNUM)." ; exit 1 ; }
ECOUNT=${#ENVS[*]}
[[ $ECOUNT == 1 ]] && EOUT=$ENVS || EOUT="${ENVS[@]}"
echo "$MB ($BNUM): [ $EOUT ]"
2 changes: 1 addition & 1 deletion buildroot/bin/mftest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ if ((AUTO_BUILD)); then
fi
else
echo "Detected \"$BDESC\" | $MB ($BNUM)."
[[ $CHOICE > $ECOUNT ]] && { echo "Environment selection out of range." ; exit 1 ; }
[[ $CHOICE > $ECOUNT ]] && { echo "Environment selection out of range." ; exit 66 ; }
fi
TARGET="${ENVS[$CHOICE-1]}"
if [[ $MB == 'SIMULATED' && $TARGET == 'linux_native' ]]; then
Expand Down
1 change: 1 addition & 0 deletions buildroot/share/git/mfhelp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Marlin Firmware Commands:
mfadd ....... Fetch a remote branch from any Marlin fork
mfclean ..... Attempt to clean up merged and deleted branches
mfdoc ....... Build the website, serve locally, and browse
mfenvs ...... Get current board SHORT_NAME (###): [ env1 env2 ... ]
mffp ........ Push new commits directly to MarlinFirmware
mfinfo ...... Provide branch information (for the other scripts)
mfinit ...... Create an 'upstream' remote for 'MarlinFirmare'
Expand Down

0 comments on commit d5dfd18

Please sign in to comment.