Merge remote-tracking branch 'remotes/stsquad/tags/pull-docker-fixes-…

…for-3.0-240718-1' into staging

docker fixes & tcg test tweak

  - graceful handling of testing under cross-compile
  - fixes for debootstrap handling
  - more helpful errors (binfmt_misc/EXECUTABLE missing)
  - drop runcom TCG test

# gpg: Signature made Tue 24 Jul 2018 11:48:32 BST
# gpg:                using RSA key FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <[email protected]>"
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-docker-fixes-for-3.0-240718-1:
  tests/tcg: remove runcom test
  docker: perform basic binfmt_misc validation in docker.py
  docker: ignore distro versioning of debootstrap
  docker: add commentary to debian-bootstrap.docker
  docker: Update debootstrap script after Debian migration from Alioth to Salsa
  docker: report hint when docker.py check fails
  docker: drop QEMU_TARGET check, fallback in EXECUTABLE not set
  docker: add expansion for docker-test-FOO to Makefile.include
  docker: add test-unit runner
  docker: Makefile.include don't include partial images
  docker: gracefully skip check_qemu
  docker: move make check into check_qemu helper
  docker: split configure_qemu from build_qemu
  docker: fail more gracefully on docker.py check
  docker: par down QEMU_CONFIGURE_OPTS in debian-tricore-cross
  docker: base debian-tricore on qemu:debian9
  tests/.gitignore: don't ignore docker tests

Signed-off-by: Peter Maydell <[email protected]>

tests/.gitignore

@@ -9,6 +9,7 @@ qht-bench
 rcutorture
 test-*
 !test-*.c
+!docker/test-*
 test-qapi-commands.[ch]
 test-qapi-events.[ch]
 test-qapi-types.[ch]

tests/docker/Makefile.include

@@ -6,7 +6,7 @@ DOCKER_SUFFIX := .docker
 DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles
 DOCKER_DEPRECATED_IMAGES := debian
 # we don't run tests on intermediate images (used as base by another image)
-DOCKER_INTERMEDIATE_IMAGES := debian8 debian9 debian8-mxe debian-ports debian-sid
+DOCKER_PARTIAL_IMAGES := debian debian8 debian9 debian8-mxe debian-ports debian-sid debian-bootstrap
 DOCKER_IMAGES := $(filter-out $(DOCKER_DEPRECATED_IMAGES),$(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker)))))
 DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
 # Use a global constant ccache directory to speed up repetitive builds
@@ -58,13 +58,11 @@ docker-image-%: $(DOCKER_FILES_DIR)/%.docker
 docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker
 	$(if $(EXECUTABLE),,\
 		$(error EXECUTABLE not set, debootstrap of debian-$* would fail))
-	$(if $(wildcard $(EXECUTABLE)),,\
-		$(error Please build $(EXECUTABLE) first))
 	$(if $(DEB_ARCH),,\
 		$(error DEB_ARCH not set, debootstrap of debian-$* would fail))
 	$(if $(DEB_TYPE),,\
 		$(error DEB_TYPE not set, debootstrap of debian-$* would fail))
-	$(if $(filter $(QEMU_TARGET),$(TARGET_DIRS)),				\
+	$(if $(wildcard $(EXECUTABLE)),						\
 		$(call quiet-command,						\
 			DEB_ARCH=$(DEB_ARCH)					\
 			DEB_TYPE=$(DEB_TYPE)					\
@@ -75,7 +73,8 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker
 			$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)), \
 			"BUILD","binfmt debian-$* (debootstrapped)"),		\
 		$(call quiet-command,						\
-			$(DOCKER_SCRIPT) check --quiet qemu:debian-$* $<,	\
+			$(DOCKER_SCRIPT) check --quiet qemu:debian-$* $< || 	\
+			{ echo "You will need to build $(EXECUTABLE)"; exit 1;},\
 			"CHECK", "debian-$* exists"))
 
 endif
@@ -121,6 +120,11 @@ docker-image-travis: NOUSER=1
 # Specialist build images, sometimes very limited tools
 docker-image-tricore-cross: docker-image-debian9
 
+# These images may be good enough for building tests but not for test builds
+DOCKER_PARTIAL_IMAGES += debian-alpha-cross debian-hppa-cross debian-m68k-cross debian-sh4-cross
+DOCKER_PARTIAL_IMAGES += debian-sparc64-cross debian-mips64-cross debian-riscv64-cross
+DOCKER_PARTIAL_IMAGES += debian-tricore-cross debian-powerpc-cross fedora-i386-cross
+
 # Rules for building linux-user powered images
 #
 # These are slower than using native cross compiler setups but can
@@ -131,19 +135,19 @@ docker-image-tricore-cross: docker-image-debian9
 # broken so we need a qemu-linux-user for this target
 docker-binfmt-image-debian-powerpc-user: DEB_ARCH = powerpc
 docker-binfmt-image-debian-powerpc-user: DEB_TYPE = jessie
-docker-binfmt-image-debian-powerpc-user: QEMU_TARGET = ppc-linux-user
 docker-binfmt-image-debian-powerpc-user: EXECUTABLE = ${BUILD_DIR}/ppc-linux-user/qemu-ppc
 docker-image-debian-powerpc-user-cross: docker-binfmt-image-debian-powerpc-user
 DOCKER_USER_IMAGES += debian-powerpc-user
 
 # Expand all the pre-requistes for each docker image and test combination
-$(foreach i,$(filter-out $(DOCKER_INTERMEDIATE_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPRECATED_IMAGES)), \
+$(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPRECATED_IMAGES)), \
 	$(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \
 		$(eval .PHONY: docker-$t@$i) \
 		$(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \
 	) \
 	$(foreach t,$(DOCKER_TESTS), \
-		$(eval docker-test: docker-$t@$i) \
+		$(eval docker-all-tests: docker-$t@$i) \
+		$(eval docker-$t: docker-$t@$i) \
 	) \
 )
 
@@ -153,7 +157,8 @@ docker:
 	@echo 'Available targets:'
 	@echo
 	@echo '    docker:              Print this help.'
-	@echo '    docker-test:         Run all image/test combinations.'
+	@echo '    docker-all-tests:    Run all image/test combinations.'
+	@echo '    docker-TEST:         Run TEST on all image combinations.'
 	@echo '    docker-clean:        Kill and remove residual docker testing containers.'
 	@echo '    docker-TEST@IMAGE:   Run "TEST" in container "IMAGE".'
 	@echo '                         Note: "TEST" is one of the listed test name,'

tests/docker/common.rc

@@ -21,7 +21,7 @@ requires()
     done
 }
 
-build_qemu()
+configure_qemu()
 {
     config_opts="--enable-werror \
                  ${TARGET_LIST:+--target-list=${TARGET_LIST}} \
@@ -32,9 +32,31 @@ build_qemu()
     echo $config_opts
     $QEMU_SRC/configure $config_opts || \
         { cat config.log && test_fail "Failed to run 'configure'"; }
+}
+
+build_qemu()
+{
+    configure_qemu $@
     make $MAKEFLAGS
 }
 
+check_qemu()
+{
+    # default to make check unless the caller specifies
+    if test -z "$@"; then
+        INVOCATION="check"
+    else
+        INVOCATION="$@"
+    fi
+
+    if command -v gtester > /dev/null 2>&1 && \
+           gtester --version > /dev/null 2>&1; then
+        make $MAKEFLAGS $INVOCATION
+    else
+        echo "No working gtester, skipping make $INVOCATION"
+    fi
+}
+
 test_fail()
 {
     echo "$@"

tests/docker/docker.py

@@ -112,6 +112,31 @@ def _copy_binary_with_libs(src, dest_dir):
             so_path = os.path.dirname(l)
             _copy_with_mkdir(l , dest_dir, so_path)
 
+
+def _check_binfmt_misc(executable):
+    """Check binfmt_misc has entry for executable in the right place.
+
+    The details of setting up binfmt_misc are outside the scope of
+    this script but we should at least fail early with a useful
+    message if it won't work."""
+
+    binary = os.path.basename(executable)
+    binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
+
+    if not os.path.exists(binfmt_entry):
+        print ("No binfmt_misc entry for %s" % (binary))
+        return False
+
+    with open(binfmt_entry) as x: entry = x.read()
+
+    qpath = "/usr/bin/%s" % (binary)
+    if not re.search("interpreter %s\n" % (qpath), entry):
+        print ("binfmt_misc for %s does not point to %s" % (binary, qpath))
+        return False
+
+    return True
+
+
 def _read_qemu_dockerfile(img_name):
     # special case for Debian linux-user images
     if img_name.startswith("debian") and img_name.endswith("user"):
@@ -315,6 +340,11 @@ def run(self, args, argv):
             # Create a docker context directory for the build
             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 
+            # Validate binfmt_misc will work
+            if args.include_executable:
+                if not _check_binfmt_misc(args.include_executable):
+                    return 1
+
             # Is there a .pre file to run in the build context?
             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
             if os.path.exists(docker_pre):
@@ -479,7 +509,12 @@ def args(self, parser):
     def run(self, args, argv):
         tag = args.tag
 
-        dkr = Docker()
+        try:
+            dkr = Docker()
+        except:
+            print("Docker not set up")
+            return 1
+
         info = dkr.inspect_tag(tag)
         if info is None:
             print("Image does not exist")

tests/docker/dockerfiles/debian-bootstrap.docker

@@ -9,6 +9,7 @@ FROM scratch
 ADD . /
 
 # Patch all mounts as docker already has stuff set up
+# (this is not needed for later debootstraps but is harmless atm)
 RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions
 
 # Run stage 2

tests/docker/dockerfiles/debian-bootstrap.pre

@@ -56,13 +56,16 @@ if [ -z $DEBOOTSTRAP_DIR ]; then
     if [ -z $DEBOOTSTRAP ]; then
         echo "No debootstrap installed, attempting to install from SCM"
         NEED_DEBOOTSTRAP=true
-    elif ! (echo "${MIN_DEBOOTSTRAP_VERSION}" ; "${DEBOOTSTRAP}" --version \
-            | cut -d ' ' -f 2) | sort -t . -n -k 1,1 -k 2,2 -k 3,3 -c &>/dev/null; then
-        echo "debootstrap too old, attempting to install from SCM"
-        NEED_DEBOOTSTRAP=true
+    else
+        INSTALLED_VERSION=$(${DEBOOTSTRAP} --version | sed 's/debootstrap \([0-9\.]*\)[^0-9\.]*.*/\1/')
+        if ! (echo "${MIN_DEBOOTSTRAP_VERSION}" ; echo "${INSTALLED_VERSION}") \
+                | sort -t . -n -k 1,1 -k 2,2 -k 3,3 -C ; then
+            echo "debootstrap too old, attempting to install from SCM"
+            NEED_DEBOOTSTRAP=true
+        fi
     fi
     if $NEED_DEBOOTSTRAP; then
-        DEBOOTSTRAP_SOURCE=https://anonscm.debian.org/git/d-i/debootstrap.git
+        DEBOOTSTRAP_SOURCE=https://salsa.debian.org/installer-team/debootstrap.git
         git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git
         export DEBOOTSTRAP_DIR=./debootstrap.git
         DEBOOTSTRAP=./debootstrap.git/debootstrap

tests/docker/dockerfiles/debian-tricore-cross.docker

@@ -7,7 +7,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 #
-FROM debian:9
+FROM qemu:debian9
 
 MAINTAINER Philippe Mathieu-Daudé <[email protected]>
 
@@ -19,5 +19,5 @@ RUN git clone --single-branch \
     make && make install && \
     rm -rf /usr/src/binutils
 
-# Specify the cross prefix for this image (see tests/docker/common.rc)
-ENV QEMU_CONFIGURE_OPTS --cross-prefix=tricore-
+# This image isn't designed for building QEMU but building tests
+ENV QEMU_CONFIGURE_OPTS --disable-system --disable-user

tests/docker/test-clang

@@ -23,5 +23,5 @@ OPTS="--cxx=clang++ --cc=clang --host-cc=clang"
 #OPTS="$OPTS --extra-cflags=-fsanitize=undefined \
     #--extra-cflags=-fno-sanitize=float-divide-by-zero"
 build_qemu $OPTS
-make $MAKEFLAGS check
+check_qemu
 install_qemu

tests/docker/test-debug

@@ -22,5 +22,5 @@ OPTS="--cxx=clang++ --cc=clang --host-cc=clang"
 OPTS="--enable-debug --enable-sanitizers $OPTS"
 
 build_qemu $OPTS
-make $MAKEFLAGS V=1 check
+check_qemu check V=1
 install_qemu

tests/docker/test-full

@@ -15,4 +15,4 @@
 
 cd "$BUILD_DIR"
 
-build_qemu && make check $MAKEFLAGS && install_qemu
+build_qemu && check_qemu && install_qemu

tests/docker/test-quick

@@ -18,5 +18,5 @@ cd "$BUILD_DIR"
 DEF_TARGET_LIST="x86_64-softmmu,aarch64-softmmu"
 TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \
 build_qemu
-make check $MAKEFLAGS
+check_qemu
 install_qemu

tests/docker/test-unit

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+#
+# Build and run the unit tests
+#
+# Copyright (c) 2018 Linaro Ltd.
+#
+# Authors:
+#  Alex Bennée <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+cd "$BUILD_DIR"
+
+# although we are not building QEMU itself we still need a configured
+# build for the unit tests to be built and run
+configure_qemu
+check_qemu check-unit

tests/tcg/i386/Makefile.target

@@ -29,11 +29,6 @@ test-i386: test-i386.c test-i386-code16.S test-i386-vm86.S test-i386.h test-i386
 	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ \
 	   $(<D)/test-i386.c $(<D)/test-i386-code16.S $(<D)/test-i386-vm86.S -lm
 
-# Specialist test runners
-run-runcom: TIMEOUT=30
-run-runcom: runcom pi_10.com
-	$(call run-test,$<,$(QEMU) ./runcom $(I386_SRC)/pi_10.com,"$< on $(TARGET_NAME)")
-
 ifeq ($(SPEED), slow)
 
 test-i386-fprem.ref: test-i386-fprem

tests/tcg/i386/README

@@ -25,9 +25,6 @@ and host CPUs.
 test-i386-fprem
 ---------------
 
-runcom
-------
-
 test-mmap
 ---------