From 34a4c3a0c3fc7856ace4e507dc49f1a9dcbcf6cb Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 23 May 2021 14:57:02 +0200 Subject: [PATCH] release script: protect bash variables and add logs Ref: https://github.com/esp8266/Arduino/issues/8049 Ref: empty variable https://github.com/esp8266/Arduino/runs/2592755965?check_suite_focus=true#step:5:35 --- package/build_boards_manager_package.sh | 135 ++++++++++++------------ 1 file changed, 70 insertions(+), 65 deletions(-) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 05778d5487..2d57bd3fdd 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -8,6 +8,7 @@ if [ ! -z "${manualversion}" ]; then plain_ver=${ver} visiblever=${ver} [ -z "${REMOTE_URL}" ] && REMOTE_URL=https://github.com/esp8266/Arduino/releases/download + echo "manual version: ver=${ver} plain_ver=${plain_ver} visiblever=${visiblever}" else @@ -16,53 +17,57 @@ else # Default to draft tag name ver=$(basename $(jq -e -r '.ref' "$GITHUB_EVENT_PATH")) # If not available, try the publish tag name - if [ "$ver" == "null" ]; then - ver=$(jq -e -r '.release.tag_name' "$GITHUB_EVENT_PATH") + if [ "${ver}" == "null" ]; then + ver=$(jq -e -r '.release.tag_name' "${GITHUB_EVENT_PATH}") + echo "release-log-1: ver=${ver} plain_ver=${plain_ver} visiblever=${visiblever}" fi # Fall back to the git description OTW (i.e. interactive) - if [ "$ver" == "null" ]; then + if [ "${ver}" == "null" ]; then ver=$(git describe --tag) + echo "release-log-2: ver=${ver} plain_ver=${plain_ver} visiblever=${visiblever}" fi - visiblever=$ver - plainver=$ver + visiblever=${ver} + plainver=${ver} + echo "release-log-3: ver=${ver} plain_ver=${plain_ver} visiblever=${visiblever}" # Match 0.0.* as special-case early-access builds if [ "${ver%.*}" = 0.0 ]; then git tag -d ${ver} ver=`git describe --tag HEAD` plain_ver=$ver + echo "release-log-4: ver=${ver} plain_ver=${plain_ver} visiblever=${visiblever}" fi fi set -e -package_name=esp8266-$visiblever -echo "Version: $visiblever ($ver)" -echo "Package name: $package_name" +package_name=esp8266-${visiblever} +echo "Version: ${visiblever} (real: ${ver})" +echo "Package name: ${package_name}" # Set REMOTE_URL environment variable to the address where the package will be # available for download. This gets written into package json file. -if [ -z "$REMOTE_URL" ]; then +if [ -z "${REMOTE_URL}" ]; then REMOTE_URL="http://localhost:8000" echo "REMOTE_URL not defined, using default" fi -echo "Remote: $REMOTE_URL" +echo "Remote: ${REMOTE_URL}" -if [ -z "$PKG_URL" ]; then - if [ -z "$PKG_URL_PREFIX" ]; then - PKG_URL_PREFIX="$REMOTE_URL/versions/$visiblever" +if [ -z "${PKG_URL}" ]; then + if [ -z "${PKG_URL_PREFIX}" ]; then + PKG_URL_PREFIX="${REMOTE_URL}/versions/${visiblever}" fi - PKG_URL="$PKG_URL_PREFIX/$package_name.zip" + PKG_URL="${PKG_URL_PREFIX}/${package_name}.zip" fi -echo "Package: $PKG_URL" -echo "Docs: $DOC_URL" +echo "Package: ${PKG_URL}" +echo "Docs: ${DOC_URL}" pushd .. # Create directory for the package -outdir=package/versions/$visiblever/$package_name -srcdir=$PWD -rm -rf package/versions/$visiblever -mkdir -p $outdir +outdir=package/versions/${visiblever}/${package_name} +srcdir=${PWD} +rm -rf package/versions/${visiblever} +mkdir -p ${outdir} # Some files should be excluded from the package cat << EOF > exclude.txt @@ -76,7 +81,7 @@ EOF # Also include all files which are ignored by git git ls-files --other --directory >> exclude.txt # Now copy files to $outdir -rsync -a --exclude-from 'exclude.txt' $srcdir/ $outdir/ +rsync -a --exclude-from 'exclude.txt' ${srcdir}/ ${outdir}/ rm exclude.txt # For compatibility, on OS X we need GNU sed which is usually called 'gsed' @@ -88,7 +93,7 @@ fi # Do some replacements in platform.txt file, which are required because IDE # handles tool paths differently when package is installed in hardware folder -cat $srcdir/platform.txt | \ +cat ${srcdir}/platform.txt | \ $SED 's/runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-lx106-elf//g' | \ $SED 's/runtime.tools.python3.path=.*//g' | \ $SED 's/runtime.tools.esptool.path={runtime.platform.path}\/tools\/esptool//g' | \ @@ -99,54 +104,54 @@ $SED 's/^#tools.esptool.cmd=/tools.esptool.cmd=/g' | \ $SED 's/^#tools.esptool.network_cmd=/tools.esptool.network_cmd=/g' | \ $SED 's/tools.mkspiffs.path={runtime.platform.path}\/tools\/mkspiffs/tools.mkspiffs.path=\{runtime.tools.mkspiffs.path\}/g' |\ $SED 's/recipe.hooks.*makecorever.*//g' |\ -$SED "s/version=.*/version=$ver/g" |\ -$SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1($ver)/g"\ - > $outdir/platform.txt +$SED "s/version=.*/version=${ver}/g" |\ +$SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1(${ver})/g"\ + > ${outdir}/platform.txt # Put core version and short hash of git version into core_version.h -ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"` -echo Ver define: $ver_define -echo \#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp8266/core_version.h -echo \#define ARDUINO_ESP8266_GIT_DESC `git describe --tags 2>/dev/null` >>$outdir/cores/esp8266/core_version.h -echo \#define ARDUINO_ESP8266_RELEASE_$ver_define >>$outdir/cores/esp8266/core_version.h -echo \#define ARDUINO_ESP8266_RELEASE \"$ver_define\" >>$outdir/cores/esp8266/core_version.h +ver_define=`echo "${plain_ver}" | tr "[:lower:]." "[:upper:]_"` +echo "ver_define: ${ver_define} (plain_ver: ${plain_ver})" +echo "#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null`" >${outdir}/cores/esp8266/core_version.h +echo "#define ARDUINO_ESP8266_GIT_DESC `git describe --tags 2>/dev/null`" >>${outdir}/cores/esp8266/core_version.h +echo "#define ARDUINO_ESP8266_RELEASE_${ver_define}" >>${outdir}/cores/esp8266/core_version.h +echo "#define ARDUINO_ESP8266_RELEASE \"${ver_define}\"" >>${outdir}/cores/esp8266/core_version.h # Zip the package -pushd package/versions/$visiblever -echo "Making $package_name.zip" -zip -qr $package_name.zip $package_name -rm -rf $package_name +pushd package/versions/${visiblever} +echo "Making ${package_name}.zip" +zip -qr ${package_name}.zip ${package_name} +rm -rf ${package_name} # Calculate SHA sum and size -sha=`shasum -a 256 $package_name.zip | cut -f 1 -d ' '` -size=`/bin/ls -l $package_name.zip | awk '{print $5}'` -echo Size: $size -echo SHA-256: $sha +sha=`shasum -a 256 ${package_name}.zip | cut -f 1 -d ' '` +size=`/bin/ls -l ${package_name}.zip | awk '{print $5}'` +echo "Size: ${size}" +echo "SHA-256: ${sha}" echo "Making package_esp8266com_index.json" -jq_arg=".packages[0].platforms[0].version = \"$visiblever\" | \ - .packages[0].platforms[0].url = \"$PKG_URL\" |\ - .packages[0].platforms[0].archiveFileName = \"$package_name.zip\"" +jq_arg=".packages[0].platforms[0].version = \"${visiblever}\" | \ + .packages[0].platforms[0].url = \"${PKG_URL}\" |\ + .packages[0].platforms[0].archiveFileName = \"${package_name}.zip\"" -if [ -z "$is_nightly" ]; then - jq_arg="$jq_arg |\ - .packages[0].platforms[0].size = \"$size\" |\ - .packages[0].platforms[0].checksum = \"SHA-256:$sha\"" +if [ -z "${is_nightly}" ]; then + jq_arg="${jq_arg} |\ + .packages[0].platforms[0].size = \"${size}\" |\ + .packages[0].platforms[0].checksum = \"SHA-256:${sha}\"" fi -if [ ! -z "$DOC_URL" ]; then - jq_arg="$jq_arg |\ - .packages[0].platforms[0].help.online = \"$DOC_URL\"" +if [ ! -z "${DOC_URL}" ]; then + jq_arg="${jq_arg} |\ + .packages[0].platforms[0].help.online = \"${DOC_URL}\"" fi -cat $srcdir/package/package_esp8266com_index.template.json | \ - jq "$jq_arg" > package_esp8266com_index.json +cat ${srcdir}/package/package_esp8266com_index.template.json | \ + jq "${jq_arg}" > package_esp8266com_index.json # Use Github API token, if available curl_gh_token_arg=() -if [ ! -z "$CI_GITHUB_API_KEY" ]; then - curl_gh_token_arg=(-H "Authorization: token $CI_GITHUB_API_KEY") +if [ ! -z "${CI_GITHUB_API_KEY}" ]; then + curl_gh_token_arg=(-H "Authorization: token ${CI_GITHUB_API_KEY}") fi # Get previous release name curl --silent "${curl_gh_token_arg[@]}" https://api.github.com/repos/esp8266/Arduino/releases > releases.json @@ -157,39 +162,39 @@ prev_any_release=$(jq -r '. | map(select(.draft == false)) | sort_by(.created_at # Previous pre-release prev_pre_release=$(jq -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' releases.json) -echo "Previous release: $prev_release" -echo "Previous (pre-?)release: $prev_any_release" -echo "Previous pre-release: $prev_pre_release" +echo "Previous release: ${prev_release}" +echo "Previous (pre-?)release: ${prev_any_release}" +echo "Previous pre-release: ${prev_pre_release}" # Make all released versions available in one package (i.e. don't separate stable/staging versions) -base_ver=$prev_any_release +base_ver=${prev_any_release} # Download previous release -echo "Downloading base package: $base_ver" +echo "Downloading base package: ${base_ver}" old_json=package_esp8266com_index_stable.json -curl -L -o $old_json "https://github.com/esp8266/Arduino/releases/download/${base_ver}/package_esp8266com_index.json" +curl -L -o ${old_json} "https://github.com/esp8266/Arduino/releases/download/${base_ver}/package_esp8266com_index.json" new_json=package_esp8266com_index.json set +e # Merge the old and new -python3 ../../merge_packages.py $new_json $old_json > tmp +python3 ../../merge_packages.py ${new_json} ${old_json} > tmp # additional json to merge (for experimental releases) echo "Additional json package files: ${MOREJSONPACKAGES}" for json in ${MOREJSONPACKAGES}; do - if [ ! -z "$json" -a -r "$json" ]; then - echo "- merging $json" - python3 ../../merge_packages.py tmp $json > tmp2 + if [ ! -z "${json}" -a -r "${json}" ]; then + echo "- merging ${json}" + python3 ../../merge_packages.py tmp ${json} > tmp2 mv tmp2 tmp fi done # drop any obsolete package versions -python3 ../../drop_versions.py - tools 1.20.0-26-gb404fb9 < tmp > tmp2 && mv tmp2 $new_json && rm $old_json tmp +python3 ../../drop_versions.py - tools 1.20.0-26-gb404fb9 < tmp > tmp2 && mv tmp2 ${new_json} && rm ${old_json} tmp # Verify the JSON file can be read, fail if it's not OK set -e -cat $new_json | jq empty +cat ${new_json} | jq empty popd popd