Add linting for shell scripts and cleanup shell scripts

CMakeLists.txt

@@ -89,6 +89,7 @@ if(ENABLE_TEST_DEPENDENCIES)
   include(${CMAKE_SOURCE_DIR}/build/cmake/test/unbound.cmake)
   include(${CMAKE_SOURCE_DIR}/build/cmake/test/mongo-orchestration.cmake)
   include(${CMAKE_SOURCE_DIR}/build/cmake/test/lua-deps.cmake)
+  include(${CMAKE_SOURCE_DIR}/build/cmake/test/shellcheck.cmake)
 endif()
 include(${CMAKE_SOURCE_DIR}/build/cmake/test/lint.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/test/test-proxy.cmake)

build/cmake/test/lint.cmake

@@ -6,8 +6,14 @@ add_custom_target(
   VERBATIM
 )
 
+add_custom_target(
+  shell-lint-target
+  COMMAND ${CMAKE_SOURCE_DIR}/test/scripts/shell-lint
+)
+
 add_custom_target(
   lint
   COMMAND ${CMAKE_BUILD_TOOL} all
   COMMAND ${CMAKE_BUILD_TOOL} lint-target
+  COMMAND ${CMAKE_BUILD_TOOL} shell-lint-target
 )

build/cmake/test/shellcheck.cmake

@@ -0,0 +1,11 @@
+ExternalProject_Add(
+  shellcheck
+  URL https://github.com/koalaman/shellcheck/archive/v${SHELLCHECK_VERSION}.tar.gz
+  URL_HASH MD5=${SHELLCHECK_HASH}
+  BUILD_IN_SOURCE 1
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND cabal sandbox init
+    COMMAND cabal update
+    COMMAND cabal install --reinstall --disable-library-profiling --disable-profiling --disable-optimization --disable-tests --disable-coverage --disable-benchmarks --disable-documentation
+  INSTALL_COMMAND cp <SOURCE_DIR>/.cabal-sandbox/bin/shellcheck ${TEST_INSTALL_PREFIX}/bin/
+)

build/cmake/versions.cmake

@@ -75,6 +75,8 @@ set(RSYSLOG_VERSION 8.14.0)
 set(RSYSLOG_HASH 443b5b1d2b84f5cd429d06d230af7fb2352336fa6449cb6484dbd4418a7ae7c2)
 set(RUNIT_VERSION 2.1.2)
 set(RUNIT_HASH 6c985fbfe3a34608eb3c53dc719172c4)
+set(SHELLCHECK_VERSION 0.4.3)
+set(SHELLCHECK_HASH 27bacd94775b62c654a3a2f753da576e)
 # Don't move to TrafficServer 6 until we can verify keepalive behavior:
 # https://issues.apache.org/jira/browse/TS-3959
 set(TRAFFICSERVER_VERSION 5.3.2)

build/package/build_package

@@ -2,72 +2,64 @@
 
 set -e -u -x
 
-SOURCE_DIR="$(dirname $(dirname $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)))"
+source_dir="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")"
 
-VERSION_STRING=$(cat $SOURCE_DIR/src/api-umbrella/version.txt)
-VERSION_MATCHER="^([^-]+)(-(.+))?$"
-if [[ $VERSION_STRING =~ $VERSION_MATCHER ]]; then
-  VERSION=${BASH_REMATCH[1]}
-  VERSION_PRE=${BASH_REMATCH[3]}
-fi
-
-PACKAGE_ITERATION="1"
-if [ -n "$VERSION_PRE" ]; then
-  PACKAGE_ITERATION="0.$PACKAGE_ITERATION.$VERSION_PRE"
-fi
+# shellcheck source=build/package/parse_version
+source "$source_dir/build/package/parse_version"
 
-source $SOURCE_DIR/build/package_dependencies.sh
+# shellcheck source=build/package_dependencies.sh
+source "$source_dir/build/package_dependencies.sh"
 
 if [ -f /etc/redhat-release ]; then
-  PACKAGE_TYPE="rpm"
-  RPM_DIST=$(rpm --query centos-release | grep -o "el[0-9]")
+  package_type="rpm"
+  rpm_dist=$(rpm --query centos-release | grep -o "el[0-9]")
 elif [ -f /etc/debian_version ]; then
-  PACKAGE_TYPE="deb"
-  PACKAGE_ITERATION="$PACKAGE_ITERATION~$(lsb_release --codename --short)"
+  package_type="deb"
+  package_iteration="$package_iteration~$(lsb_release --codename --short)"
 else
   echo "Unknown build system"
   exit 1
 fi
 
-FPM_ARGS=()
-FPM_ARGS+=("--rpm-dist" "$RPM_DIST")
-FPM_ARGS+=("-t" "$PACKAGE_TYPE")
-FPM_ARGS+=("-s" "dir")
-FPM_ARGS+=("--verbose")
-FPM_ARGS+=("--license" "MIT")
-FPM_ARGS+=("--version" "$VERSION")
-FPM_ARGS+=("--iteration" "$PACKAGE_ITERATION")
-if [ "$PACKAGE_TYPE" == "rpm" ]; then
-  FPM_ARGS+=("--rpm-dist" "$RPM_DIST")
-  FPM_ARGS+=("--rpm-compression" "xz")
-elif [ "$PACKAGE_TYPE" == "deb" ]; then
-  FPM_ARGS+=("--deb-compression" "xz")
-  FPM_ARGS+=("--deb-no-default-config-files")
+fpm_args=()
+fpm_args+=("--rpm-dist" "$rpm_dist")
+fpm_args+=("-t" "$package_type")
+fpm_args+=("-s" "dir")
+fpm_args+=("--verbose")
+fpm_args+=("--license" "MIT")
+fpm_args+=("--version" "$version")
+fpm_args+=("--iteration" "$package_iteration")
+if [ "$package_type" == "rpm" ]; then
+  fpm_args+=("--rpm-dist" "$rpm_dist")
+  fpm_args+=("--rpm-compression" "xz")
+elif [ "$package_type" == "deb" ]; then
+  fpm_args+=("--deb-compression" "xz")
+  fpm_args+=("--deb-no-default-config-files")
 fi
 
 if [ "$PACKAGE" == "core" ]; then
-  for DEP in "${CORE_PACKAGE_DEPENDENCIES[@]}"; do
-    FPM_ARGS+=("-d" "$DEP")
+  for dep in "${core_package_dependencies[@]}"; do
+    fpm_args+=("-d" "$dep")
   done
 
-  FPM_ARGS+=("-C" "$WORK_DIR/package-dest-core")
-  FPM_ARGS+=("--name api-umbrella")
-  FPM_ARGS+=("--config-files" "etc/api-umbrella/api-umbrella.yml")
-  FPM_ARGS+=("--after-install" "$SOURCE_DIR/build/package/scripts/after-install")
-  FPM_ARGS+=("--before-remove" "$SOURCE_DIR/build/package/scripts/before-remove")
-  FPM_ARGS+=("--after-remove" "$SOURCE_DIR/build/package/scripts/after-remove")
-  FPM_ARGS+=("--directories" "/opt/api-umbrella")
-  FPM_ARGS+=("--directories" "/etc/api-umbrella")
+  fpm_args+=("-C" "$WORK_DIR/package-dest-core")
+  fpm_args+=("--name api-umbrella")
+  fpm_args+=("--config-files" "etc/api-umbrella/api-umbrella.yml")
+  fpm_args+=("--after-install" "$source_dir/build/package/scripts/after-install")
+  fpm_args+=("--before-remove" "$source_dir/build/package/scripts/before-remove")
+  fpm_args+=("--after-remove" "$source_dir/build/package/scripts/after-remove")
+  fpm_args+=("--directories" "/opt/api-umbrella")
+  fpm_args+=("--directories" "/etc/api-umbrella")
 elif [ "$PACKAGE" == "hadoop-analytics" ]; then
-  for DEP in "${HADOOP_ANALYTICS_PACKAGE_DEPENDENCIES[@]}"; do
-    FPM_ARGS+=("-d" "$DEP")
+  for dep in "${hadoop_analytics_package_dependencies[@]}"; do
+    fpm_args+=("-d" "$dep")
   done
 
-  FPM_ARGS+=("-C" "$WORK_DIR/package-dest-hadoop-analytics")
-  FPM_ARGS+=("--name api-umbrella-hadoop-analytics")
-  FPM_ARGS+=("--depends" "api-umbrella")
+  fpm_args+=("-C" "$WORK_DIR/package-dest-hadoop-analytics")
+  fpm_args+=("--name api-umbrella-hadoop-analytics")
+  fpm_args+=("--depends" "api-umbrella")
 fi
 
-mkdir -p $WORK_DIR/package/build/$PACKAGE
-cd $WORK_DIR/package/build/$PACKAGE
-env XZ_OPT=-9 bundle exec fpm ${FPM_ARGS[*]}
+mkdir -p "$WORK_DIR/package/build/$PACKAGE"
+cd "$WORK_DIR/package/build/$PACKAGE"
+env XZ_OPT=-9 bundle exec fpm "${fpm_args[@]}"

build/package/docker_run

@@ -2,27 +2,27 @@
 
 set -e -u -x
 
-SOURCE_DIR="$(dirname $(dirname $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)))"
-DOCKER_IMAGE=${DIST/-/:}
-CONTAINER_NAME="api-umbrella-package-$DIST"
-CONTAINER_NAME=${CONTAINER_NAME//[^a-zA-Z0-9_.-]/}
+source_dir="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")"
+docker_image=${DIST/-/:}
+container_name="api-umbrella-package-$DIST"
+container_name=${container_name//[^a-zA-Z0-9_.-]/}
 
 # Make sure we have the latest base image.
-docker pull $DOCKER_IMAGE
+docker pull "$docker_image"
 
 # Wipe any previous build attempts.
-EXISTING=$(docker ps -a -q --filter="name=$CONTAINER_NAME")
-if [ "$EXISTING" != "" ]; then
-  docker rm -f $EXISTING
+existing=$(docker ps -a -q --filter="name=$container_name")
+if [ "$existing" != "" ]; then
+  docker rm -f "$existing"
 fi
 
 # Create the package inside the docker container for this distro.
 docker run \
-  --name=$CONTAINER_NAME \
-  --volume=$SOURCE_DIR:/api-umbrella \
+  --name="$container_name" \
+  --volume="$source_dir:/api-umbrella" \
   --env="DIST=$DIST" \
   --rm \
-  $DOCKER_IMAGE \
+  "$docker_image" \
   /bin/bash -c "/api-umbrella/build/package/docker_script"
 
 echo "Completed package build for $DIST"

build/package/docker_script

@@ -2,27 +2,27 @@
 
 set -e -u -x
 
-SOURCE_DIR="$(dirname $(dirname $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)))"
+source_dir="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")"
 
 # Install any system dependencies for building.
-$SOURCE_DIR/build/scripts/install_build_dependencies
+"$source_dir/build/scripts/install_build_dependencies"
 
 # Wipe any cmake files in the source directory, so we can perform an
 # out-of-source build.
-rm -rf $SOURCE_DIR/cmake_install.cmake $SOURCE_DIR/CMakeCache.txt $SOURCE_DIR/CMakeFiles $SOURCE_DIR/Makefile
+rm -rf "$source_dir/cmake_install.cmake" "$source_dir/CMakeCache.txt" "$source_dir/CMakeFiles" "$source_dir/Makefile"
 
 # Perform an out-of-source build (so the same source directory can be used for
 # multiple parallel docker builds).
-BUILD_DIR=/tmp/api-umbrella-$DIST
-rm -rf $BUILD_DIR
-mkdir -p $BUILD_DIR
-cd $BUILD_DIR
-$SOURCE_DIR/configure --enable-hadoop-analytics
+build_dir="/tmp/api-umbrella-$DIST"
+rm -rf "$build_dir"
+mkdir -p "$build_dir"
+cd "$build_dir"
+"$source_dir/configure" --enable-hadoop-analytics
 make
 make package
 
 # Copy the packages for this distro back to the source directory so it's
 # available on the host machine, outside the docker build container.
-rm -rf $SOURCE_DIR/build/work/package/current/$DIST
-mkdir -p $SOURCE_DIR/build/work/package/current/$DIST
-cp -r $BUILD_DIR/build/work/package/build/* $SOURCE_DIR/build/work/package/current/$DIST/
+rm -rf "$source_dir/build/work/package/current/$DIST"
+mkdir -p "$source_dir/build/work/package/current/$DIST"
+cp -r "$build_dir/build/work/package/build"/* "$source_dir/build/work/package/current/$DIST/"

build/package/files/etc/init.d/api-umbrella

@@ -16,67 +16,71 @@
 # Short-Description: API Umbrella
 ### END INIT INFO
 
-NAME=api-umbrella
-DESC="API Umbrella"
+name=api-umbrella
+desc="API Umbrella"
 
 if [ -f /etc/rc.d/init.d/functions ]; then
+  # shellcheck disable=SC1091
   . /etc/rc.d/init.d/functions
 elif [ -f /lib/lsb/init-functions ]; then
+  # shellcheck disable=SC1091
   . /lib/lsb/init-functions
 fi
 
 if [ -f /etc/sysconfig/api-umbrella ]; then
+  # shellcheck disable=SC1091
   . /etc/sysconfig/api-umbrella
 elif [ -f /etc/default/api-umbrella ]; then
+  # shellcheck disable=SC1091
   . /etc/default/api-umbrella
 fi
 
 start() {
   if type log_daemon_msg > /dev/null 2>&1; then
-    log_daemon_msg "Starting $DESC" "$NAME"
+    log_daemon_msg "Starting $desc" "$name"
   else
-    echo -n $"Starting $NAME: "
+    echo -n $"Starting $name: "
   fi
 
   api-umbrella start
-  RETVAL=$?
+  retval=$?
 
   if type log_end_msg > /dev/null 2>&1; then
-    log_end_msg $RETVAL
+    log_end_msg $retval
   elif type success > /dev/null 2>&1; then
-    if [ $RETVAL -eq 0 ]; then
-      success $"$NAME startup"
+    if [ $retval -eq 0 ]; then
+      success $"$name startup"
     else
-      failure $"$NAME startup"
+      failure $"$name startup"
     fi
     echo
   fi
 
-  return $RETVAL
+  return $retval
 }
 
 stop() {
   if type log_daemon_msg > /dev/null 2>&1; then
-    log_daemon_msg "Stopping $DESC" "$NAME"
+    log_daemon_msg "Stopping $desc" "$name"
   else
-    echo -n $"Stopping $NAME: "
+    echo -n $"Stopping $name: "
   fi
 
   api-umbrella stop
-  RETVAL=$?
+  retval=$?
 
   if type log_end_msg > /dev/null 2>&1; then
-    log_end_msg $RETVAL
+    log_end_msg $retval
   elif type success > /dev/null 2>&1; then
-    if [ $RETVAL -eq 0 ]; then
-      success $"$NAME shutdown"
+    if [ $retval -eq 0 ]; then
+      success $"$name shutdown"
     else
-      failure $"$NAME shutdown"
+      failure $"$name shutdown"
     fi
     echo
   fi
 
-  return $RETVAL
+  return $retval
 }
 
 restart() {
@@ -86,32 +90,32 @@ restart() {
 
 reload() {
   if type log_daemon_msg > /dev/null 2>&1; then
-    log_daemon_msg "Reloading $DESC" "$NAME"
+    log_daemon_msg "Reloading $desc" "$name"
   else
-    echo -n $"Reloading $NAME: "
+    echo -n $"Reloading $name: "
   fi
 
   api-umbrella reload
-  RETVAL=$?
+  retval=$?
 
   if type log_end_msg > /dev/null 2>&1; then
-    log_end_msg $RETVAL
+    log_end_msg $retval
   elif type success > /dev/null 2>&1; then
-    if [ $RETVAL -eq 0 ]; then
-      success $"$NAME reload"
+    if [ $retval -eq 0 ]; then
+      success $"$name reload"
     else
-      failure $"$NAME reload"
+      failure $"$name reload"
     fi
     echo
   fi
 
-  return $RETVAL
+  return $retval
 }
 
 status() {
   api-umbrella status
-  RETVAL=$?
-  return $RETVAL
+  retval=$?
+  return $retval
 }
 
 status_quiet() {
@@ -139,7 +143,7 @@ case "$1" in
     restart
     ;;
   *)
-    echo "Usage: $NAME {start|stop|status|reload|restart|condrestart}"
+    echo "Usage: $name {start|stop|status|reload|restart|condrestart}"
     exit 1
     ;;
 esac

build/package/parse_version

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e -u -x
+
+source_dir="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")"
+
+version_string=$(cat "$source_dir/src/api-umbrella/version.txt")
+version_matcher="^([^-]+)(-(.+))?$"
+if [[ $version_string =~ $version_matcher ]]; then
+  # shellcheck disable=SC2034
+  version=${BASH_REMATCH[1]}
+  version_pre=${BASH_REMATCH[3]}
+fi
+
+package_iteration="1"
+if [ -n "$version_pre" ]; then
+  package_iteration="0.$package_iteration.$version_pre"
+fi