Merge pull request #8873 from daverodgman/quietbuild-2.28

Make builds less verbose - 2.28 backport

scripts/generate_visualc_files.pl

@@ -168,7 +168,7 @@ sub gen_app {
 }
 
 sub get_app_list {
-    my $app_list = `cd $programs_dir && make list`;
+    my $app_list = `cd $programs_dir && VERBOSE_LOGS=1 make list`;
     die "make list failed: $!\n" if $?;
 
     return split /\s+/, $app_list;

tests/scripts/all.sh

@@ -195,10 +195,21 @@ pre_initialize_variables () {
     # defined in this script whose name starts with "component_".
     ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//')
 
-    # Delay determinig SUPPORTED_COMPONENTS until the command line options have a chance to override
+    # Delay determining SUPPORTED_COMPONENTS until the command line options have a chance to override
     # the commands set by the environment
 }
 
+setup_quiet_wrappers()
+{
+    # Pick up "quiet" wrappers for make and cmake, which don't output very much
+    # unless there is an error. This reduces logging overhead in the CI.
+    #
+    # Note that the cmake wrapper breaks unless we use an absolute path here.
+    if [[ -e ${PWD}/tests/scripts/quiet ]]; then
+        export PATH=${PWD}/tests/scripts/quiet:$PATH
+    fi
+}
+
 # Test whether the component $1 is included in the command line patterns.
 is_component_included()
 {
@@ -997,6 +1008,9 @@ EOF
 }
 
 component_test_zlib_cmake() {
+    # This is needed due to something parsing the output from make
+    export VERBOSE_LOGS=1
+
     msg "build: zlib enabled, cmake"
     scripts/config.py set MBEDTLS_ZLIB_SUPPORT
     cmake -D ENABLE_ZLIB_SUPPORT=On -D CMAKE_BUILD_TYPE:String=Release .
@@ -3733,6 +3747,7 @@ pre_check_environment
 pre_initialize_variables
 pre_parse_command_line "$@"
 
+setup_quiet_wrappers
 pre_check_git
 pre_restore_files
 pre_back_up

tests/scripts/check_files.py

@@ -172,6 +172,8 @@ class ShebangIssueTracker(FileIssueTracker):
         b'sh': 'sh',
     }
 
+    path_exemptions = re.compile(r'tests/scripts/quiet/.*')
+
     def is_valid_shebang(self, first_line, filepath):
         m = re.match(self._shebang_re, first_line)
         if not m:

tests/scripts/quiet/cmake

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+#
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+#
+# This swallows the output of the wrapped tool, unless there is an error.
+# This helps reduce excess logging in the CI.
+
+# If you are debugging a build / CI issue, you can get complete unsilenced logs
+# by un-commenting the following line (or setting VERBOSE_LOGS in your environment):
+
+# export VERBOSE_LOGS=1
+
+# don't silence invocations containing these arguments
+export NO_SILENCE=" --version "
+
+export TOOL="cmake"
+
+exec "$(dirname "$0")/quiet.sh" "$@"

tests/scripts/quiet/make

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+#
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+#
+# This swallows the output of the wrapped tool, unless there is an error.
+# This helps reduce excess logging in the CI.
+
+# If you are debugging a build / CI issue, you can get complete unsilenced logs
+# by un-commenting the following line (or setting VERBOSE_LOGS in your environment):
+
+# export VERBOSE_LOGS=1
+
+# don't silence invocations containing these arguments
+export NO_SILENCE=" --version | test "
+
+export TOOL="make"
+
+exec "$(dirname "$0")/quiet.sh" "$@"

tests/scripts/quiet/quiet.sh

@@ -0,0 +1,75 @@
+# -*-mode: sh; sh-shell: bash -*-
+#
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+#
+# This swallows the output of the wrapped tool, unless there is an error.
+# This helps reduce excess logging in the CI.
+
+# If you are debugging a build / CI issue, you can get complete unsilenced logs
+# by un-commenting the following line (or setting VERBOSE_LOGS in your environment):
+#
+# VERBOSE_LOGS=1
+#
+# This script provides most of the functionality for the adjacent make and cmake
+# wrappers.
+#
+# It requires two variables to be set:
+#
+# TOOL       - the name of the tool that is being wrapped (with no path), e.g. "make"
+#
+# NO_SILENCE - a regex that describes the commandline arguments for which output will not
+#              be silenced, e.g. " --version | test ". In this example, "make lib test" will
+#              not be silent, but "make lib" will be.
+
+# Locate original tool
+TOOL_WITH_PATH=$(dirname "$0")/$TOOL
+ORIGINAL_TOOL=$(type -ap "${TOOL}" | grep -v -Fx "$TOOL_WITH_PATH" | head -n1)
+
+print_quoted_args() {
+    # similar to printf '%q' "$@"
+    # but produce more human-readable results for common/simple cases like "a b"
+    for a in "$@"; do
+        # Get bash to quote the string
+        printf -v q '%q' "$a"
+        simple_pattern="^([-[:alnum:]_+./:@]+=)?([^']*)$"
+        if [[ "$a" != "$q" && $a =~ $simple_pattern ]]; then
+            # a requires some quoting (a != q), but has no single quotes, so we can
+            # simplify the quoted form - e.g.:
+            #   a b        -> 'a b'
+            #   CFLAGS=a b -> CFLAGS='a b'
+            q="${BASH_REMATCH[1]}'${BASH_REMATCH[2]}'"
+        fi
+        printf " %s" "$q"
+    done
+}
+
+if [[ ! " $* " =~ " --version " ]]; then
+    # Display the command being invoked - if it succeeds, this is all that will
+    # be displayed. Don't do this for invocations with --version, because
+    # this output is often parsed by scripts, so we don't want to modify it.
+    printf %s "${TOOL}"    1>&2
+    print_quoted_args "$@" 1>&2
+    echo                   1>&2
+fi
+
+if [[ " $@ " =~ $NO_SILENCE || -n "${VERBOSE_LOGS}" ]]; then
+    # Run original command with no output supression
+    exec "${ORIGINAL_TOOL}" "$@"
+else
+    # Run original command and capture output & exit status
+    TMPFILE=$(mktemp "quiet-${TOOL}.XXXXXX")
+    "${ORIGINAL_TOOL}" "$@" > "${TMPFILE}" 2>&1
+    EXIT_STATUS=$?
+
+    if [[ $EXIT_STATUS -ne 0 ]]; then
+        # On error, display the full output
+        cat "${TMPFILE}"
+    fi
+
+    # Remove tmpfile
+    rm "${TMPFILE}"
+
+    # Propagate the exit status
+    exit $EXIT_STATUS
+fi