Cirrus: Produce and collect varlink output

When executing 'make remotesystem' testing, a varlink process is started
up but it's stdio is dumped due to the production of excessive data.
However, this also means if the process has a problem, any errors will
not be accessible.

Instead, grab only the last 100 lines and direct them into a file.  Also
update automation's log collection to retrieve this file when the
`$REMOTE_CLIENT` env. var. is `true`.

Signed-off-by: Chris Evich <[email protected]>

.cirrus.yml

@@ -368,6 +368,7 @@ testing_task:
         df_script: '$SCRIPT_BASE/logcollector.sh df'
         audit_log_script: '$SCRIPT_BASE/logcollector.sh audit'
         journal_script: '$SCRIPT_BASE/logcollector.sh journal'
+        varlink_script: '$SCRIPT_BASE/logcollector.sh varlink'
 
 # Test crun only on latest Fedora
 testing_crun_task:

Makefile

@@ -89,6 +89,9 @@ RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER)
 RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH)
 RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME)
 
+# If non-empty, logs all output from varlink during remote system testing
+VARLINK_LOG ?=
+
 # If GOPATH not specified, use one in the local directory
 ifeq ($(GOPATH),)
 export GOPATH := $(CURDIR)/_output
@@ -274,7 +277,7 @@ remotesystem:
 	if timeout -v 1 true; then \
 		SOCK_FILE=$(shell mktemp --dry-run --tmpdir io.podman.XXXXXX);\
 		export PODMAN_VARLINK_ADDRESS=unix:$$SOCK_FILE; \
-		./bin/podman varlink --timeout=0 $$PODMAN_VARLINK_ADDRESS &>/dev/null & \
+		./bin/podman varlink --timeout=0 $$PODMAN_VARLINK_ADDRESS &> $(if $(VARLINK_LOG),$(VARLINK_LOG),/dev/null) & \
 		retry=5;\
 		while [[ $$retry -ge 0 ]]; do\
 			echo Waiting for varlink server...;\

contrib/cirrus/integration_test.sh

@@ -4,7 +4,7 @@ set -e
 
 source $(dirname $0)/lib.sh
 
-req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME
+req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG
 
 # Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is
 # the test suite to run; currently (2019-05) the only option is 'integration'
@@ -59,7 +59,7 @@ case "$SPECIALMODE" in
         make test-binaries
         if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
         then
-            make remote${TESTSUITE}
+            make remote${TESTSUITE} VARLINK_LOG=$VARLINK_LOG
         else
             make local${TESTSUITE}
         fi

contrib/cirrus/lib.sh

@@ -35,6 +35,9 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
 SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
 AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
 
+# Log remote-client system test varlink output here
+export VARLINK_LOG=/var/tmp/varlink.log
+
 cd $GOSRC
 if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]]
 then

contrib/cirrus/logcollector.sh

@@ -4,7 +4,7 @@ set -e
 
 source $(dirname $0)/lib.sh
 
-req_env_var CIRRUS_WORKING_DIR OS_RELEASE_ID
+req_env_var CIRRUS_WORKING_DIR OS_RELEASE_ID TEST_REMOTE_CLIENT
 
 # Assume there are other log collection commands to follow - Don't
 # let one break another that may be useful, but also keep any
@@ -32,6 +32,15 @@ case $1 in
     df) showrun df -lhTx tmpfs ;;
     ginkgo) showrun cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log ;;
     journal) showrun journalctl -b ;;
+    varlink)
+       if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
+       then
+          echo "(Trailing 100 lines of $VARLINK_LOG)"
+          showrun tail -100 $VARLINK_LOG
+       else
+          die 0 "\$TEST_REMOTE_CLIENT is not 'true': $TEST_REMOTE_CLIENT"
+       fi
+       ;;
     packages)
         # These names are common to Fedora and Ubuntu
         PKG_NAMES=(\