diff --git a/.github/workflows/ci-aarchxx.yml b/.github/workflows/ci-aarchxx.yml index 8150bc3245b..a234cceb48c 100644 --- a/.github/workflows/ci-aarchxx.yml +++ b/.github/workflows/ci-aarchxx.yml @@ -1,5 +1,5 @@ # ********************************************************** -# Copyright (c) 2020 Google, Inc. All rights reserved. +# Copyright (c) 2020-2021 Google, Inc. All rights reserved. # ********************************************************** # Redistribution and use in source and binary forms, with or without @@ -48,9 +48,10 @@ defaults: shell: bash jobs: - # AArchXX cross-compile with gcc, no tests: - aarchxx-cross-compile: - runs-on: ubuntu-16.04 + # AArch64 cross-compile with gcc, with some tests run under QEMU. + # We use a more recent Ubuntu for a more recent QEMU. + aarch64-cross-compile: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -63,15 +64,67 @@ jobs: - run: git fetch --no-tags --depth=1 origin master - # Install cross-compilers for cross-compiling Linux build: + # Install cross-compiler for cross-compiling Linux build: - name: Create Build Environment run: | sudo apt-get -y install doxygen vera++ cmake zlib1g-dev libsnappy-dev \ - g++-arm-linux-gnueabihf g++-aarch64-linux-gnu + g++-aarch64-linux-gnu qemu-user - name: Run Suite working-directory: ${{ github.workspace }} - run: ./suite/runsuite_wrapper.pl automated_ci + run: ./suite/runsuite_wrapper.pl automated_ci 64_only + env: + DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY: yes + + - name: Send failure mail to dynamorio-devs + if: failure() && github.ref == 'refs/heads/master' + uses: dawidd6/action-send-mail@v2 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_USERNAME}} + password: ${{secrets.DYNAMORIO_NOTIFICATION_EMAIL_PASSWORD}} + subject: | + [${{github.repository}}] ${{github.workflow}} FAILED + on ${{github.event_name}} at ${{github.ref}} + body: | + Github Actions CI workflow run FAILED! + Workflow: ${{github.workflow}}/aarchxx-cross-compile + Repository: ${{github.repository}} + Branch ref: ${{github.ref}} + SHA: ${{github.sha}} + Triggering actor: ${{github.actor}} + Triggering event: ${{github.event_name}} + Run Id: ${{github.run_id}} + See more details on github.com/DynamoRIO/dynamorio/actions/runs/${{github.run_id}} + to: dynamorio-devs@googlegroups.com + from: Github Action CI + + # ARM cross-compile with gcc, with some tests run under QEMU. + # We use a more recent Ubuntu for a more recent QEMU. + arm-cross-compile: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + # Cancel any prior runs for a PR (but do not cancel master branch runs). + - uses: n1hility/cancel-previous-runs@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + if: ${{ github.event_name == 'pull_request' }} + + - run: git fetch --no-tags --depth=1 origin master + + # Install cross-compiler for cross-compiling Linux build: + - name: Create Build Environment + run: | + sudo apt-get -y install doxygen vera++ cmake zlib1g-dev libsnappy-dev \ + g++-arm-linux-gnueabihf qemu-user + + - name: Run Suite + working-directory: ${{ github.workspace }} + run: ./suite/runsuite_wrapper.pl automated_ci 32_only env: DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY: yes diff --git a/CMakeLists.txt b/CMakeLists.txt index 31eb2fee834..a6aad7e96e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -295,6 +295,16 @@ if (NOT "${TARGET_ARCH}" STREQUAL "${CMAKE_SYSTEM_PROCESSOR}") endif () endif () +# Support for running cross-compiled tests under emulation. +if (CMAKE_CROSSCOMPILING AND DEFINED CMAKE_FIND_ROOT_PATH) + find_program(QEMU_BINARY qemu-${CMAKE_SYSTEM_PROCESSOR} DOC "QEMU emulation tool") + if (NOT QEMU_BINARY) + message(STATUS "Did not find qemu-${CMAKE_SYSTEM_PROCESSOR}: tests will not run") + else () + message(STATUS "Found qemu-${CMAKE_SYSTEM_PROCESSOR} for tests under emulation") + endif () +endif () + option(VMKERNEL "target VMkernel (not officially supported yet)") # high-level configurations diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 3bcf74ff6a9..c80ffff1201 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1055,7 +1055,9 @@ endif () # We can't run core unit tests when the target is not the host, just like we can't # run DR itself managing an app: DR is a same-arch system for code cache operation # and only supports target!=host for decoding and drmemtrace analysis (i#1684). -if (BUILD_TESTS AND NOT DR_HOST_NOT_TARGET) +if (BUILD_TESTS AND + ((CMAKE_CROSSCOMPILING AND DEFINED CMAKE_FIND_ROOT_PATH) OR + (NOT DR_HOST_NOT_TARGET))) add_executable(unit_tests unit_tests.c # These unit tests have been moved from the x86_code module into a new x86_code # test module that gets its own clang/gcc options for testing (-mno-vzeroupper). @@ -1129,6 +1131,9 @@ if (BUILD_TESTS AND NOT DR_HOST_NOT_TARGET) if (APPLE) set_tests_properties(unit_tests PROPERTIES LABELS OSX) endif () + if (AARCHXX) + set_tests_properties(unit_tests PROPERTIES LABELS RUNS_ON_QEMU) + endif () copy_target_to_device(unit_tests "${location_suffix}") endif () diff --git a/make/utils_exposed.cmake b/make/utils_exposed.cmake index f2e88050d4a..5417694954f 100644 --- a/make/utils_exposed.cmake +++ b/make/utils_exposed.cmake @@ -1,5 +1,5 @@ ## ********************************************************** -## Copyright (c) 2012-2020 Google, Inc. All rights reserved. +## Copyright (c) 2012-2021 Google, Inc. All rights reserved. ## ********************************************************** ## ## Redistribution and use in source and binary forms, with or without @@ -228,6 +228,15 @@ function (DynamoRIO_prefix_cmd_if_necessary cmd_out use_ats cmd_in) else () set(${cmd_out} adb shell ${cmd_in} ${ARGN} PARENT_SCOPE) endif () + elseif (CMAKE_CROSSCOMPILING AND DEFINED CMAKE_FIND_ROOT_PATH AND QEMU_BINARY) + if (use_ats) + set(${cmd_out} + "${QEMU_BINARY}@-L@${CMAKE_FIND_ROOT_PATH}@${cmd_in}${ARGN}" + PARENT_SCOPE) + else () + set(${cmd_out} ${QEMU_BINARY} -L ${CMAKE_FIND_ROOT_PATH} + ${cmd_in} ${ARGN} PARENT_SCOPE) + endif () else () set(${cmd_out} ${cmd_in} ${ARGN} PARENT_SCOPE) endif () diff --git a/suite/runsuite.cmake b/suite/runsuite.cmake index 1d7810d02e9..792411a988f 100644 --- a/suite/runsuite.cmake +++ b/suite/runsuite.cmake @@ -1,5 +1,5 @@ # ********************************************************** -# Copyright (c) 2010-2020 Google, Inc. All rights reserved. +# Copyright (c) 2010-2021 Google, Inc. All rights reserved. # Copyright (c) 2009-2010 VMware, Inc. All rights reserved. # ********************************************************** @@ -432,6 +432,10 @@ endif (NOT cross_aarchxx_linux_only AND NOT cross_android_only AND NOT a64_on_x8 if (UNIX AND ARCH_IS_X86) # Optional cross-compilation for ARM/Linux and ARM/Android if the cross # compilers are on the PATH. + set(orig_extra_ctest_args ${extra_ctest_args}) + if (cross_aarchxx_linux_only) + set(extra_ctest_args ${extra_ctest_args} INCLUDE_LABEL RUNS_ON_QEMU) + endif () set(prev_optional_cross_compile ${optional_cross_compile}) if (NOT cross_aarchxx_linux_only) # For CI cross_aarchxx_linux_only builds, we want to fail on config failures. @@ -442,7 +446,12 @@ if (UNIX AND ARCH_IS_X86) set(ENV{CFLAGS} "") # environment vars do not obey the normal scope rules--must reset set(ENV{CXXFLAGS} "") set(prev_run_tests ${run_tests}) - set(run_tests OFF) # build tests but don't run them + if (optional_cross_compile) + find_program(QEMU_ARM_BINARY qemu-arm "QEMU emulation tool") + if (NOT QEMU_ARM_BINARY) + set(run_tests OFF) # build tests but don't run them + endif () + endif () testbuild_ex("arm-debug-internal-32" OFF " DEBUG:BOOL=ON INTERNAL:BOOL=ON @@ -454,6 +463,12 @@ if (UNIX AND ARCH_IS_X86) INTERNAL:BOOL=OFF CMAKE_TOOLCHAIN_FILE:PATH=${CTEST_SOURCE_DIRECTORY}/make/toolchain-arm32.cmake " OFF ${arg_package} "") + if (optional_cross_compile) + find_program(QEMU_AARCH64_BINARY qemu-aarch64 "QEMU emulation tool") + if (NOT QEMU_AARCH64_BINARY) + set(run_tests OFF) # build tests but don't run them + endif () + endif () testbuild_ex("arm-debug-internal-64" ON " DEBUG:BOOL=ON INTERNAL:BOOL=ON @@ -467,6 +482,7 @@ if (UNIX AND ARCH_IS_X86) " OFF ${arg_package} "") set(run_tests ${prev_run_tests}) + set(extra_ctest_args ${orig_extra_ctest_args}) set(optional_cross_compile ${prev_optional_cross_compile}) # Android cross-compilation and running of tests using "adb shell" diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index 3f5cbc11cf2..60ed4b8505b 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -1184,6 +1184,9 @@ else () set(dr_os_ops -msgbox_mask 0 -dumpcore_mask 0x7d -staged) endif () set(dr_test_ops -stderr_mask 0xC ${dr_os_ops}) +if (CMAKE_CROSSCOMPILING AND DEFINED CMAKE_FIND_ROOT_PATH) + set(dr_test_ops -xarch_root ${CMAKE_FIND_ROOT_PATH} ${dr_test_ops}) +endif () function(runtest_cmd outcmd outops key native standalone_dr dr_ops_aux separate_script) # assumes tools have been built: enforced at top level @@ -1205,6 +1208,10 @@ function(runtest_cmd outcmd outops key native standalone_dr dr_ops_aux separate_ if (DEFINED ${key}_timeout) set(timeout ${${key}_timeout}) endif () + if (CMAKE_CROSSCOMPILING) + # Everything is slower under emulation. + math(EXPR timeout "${timeout}*4") + endif () if (WIN32 OR NOT native) # Both Windows and non-native Unix tests now use the native drrun frontend. @@ -4682,3 +4689,115 @@ if (APPLE) PROPERTIES LABELS OSX) endif () endif () + +# XXX i#4719: Support for running aarchxx Linux tests under QEMU. +# Currently about 1/3 of the tests pass. The rest either are too slow under QEMU +# and hit the (expanded) timeout, or do not yet have full command line insertion +# support (drcachesim and other tools), or hit bugs in QEMU or in DR. +if (NOT ANDROID AND AARCHXX) + set_tests_properties( + code_api|common.broadfun + code_api|common.getretaddr + code_api|common.nzcv + code_api|linux.app_tls + code_api|linux.execve-null + code_api|linux.exit + code_api|linux.fib-conflict + code_api|linux.fib-conflict-early + code_api|linux.fib-pie + code_api|linux.fib-static + code_api|linux.fork + code_api|linux.infinite + code_api|linux.longjmp + code_api|linux.readlink + code_api|linux.sigaction + code_api|linux.signalfd + code_api|linux.signal_racesys + code_api|security-common.codemod + code_api|security-common.ret_noncall_trace + code_api|security-common.TestMemProtChg_FLAKY + code_api|client.app_args + code_api|client.blackbox + code_api|client.call-retarget + code_api|client.crashmsg + code_api|client.destructor + code_api|client.drcontainers-test + code_api|client.drmodtrack-test + code_api|client.drsyms-test + code_api|client.drwrap-test-callconv + code_api|client.drxmgr-test + code_api|client.exception + code_api|client.int64_overrides + code_api|client.low_on_memory + code_api|client.modules + code_api|client.null_instrument + code_api|client.option_parse + code_api|client.partial_module_map + code_api|client.stack-overflow + code_api|client.thread_exit_xl8 + code_api|sample.bbbuf + code_api|sample.bbcount + code_api|sample.bbsize + code_api|sample.div + code_api|sample.empty + code_api|sample.inline + code_api|sample.opcode_count + code_api|sample.signal + code_api|sample.stl_test + code_api|sample.syscall + code_api|sample.wrap + PROPERTIES LABELS RUNS_ON_QEMU) + if (DEBUG) + set_tests_properties( + code_api|common.logstderr + PROPERTIES LABELS RUNS_ON_QEMU) + endif () + if (AARCH64) + set_tests_properties( + code_api|common.allasm_aarch64_cache + code_api|common.allasm_aarch64_isa + code_api|linux.mmap + code_api,satisfy_w_xor_x|linux.fork + code_api|security-common.retnonexisting + code_api|security-linux.trampoline + code_api|client.app_inscount + code_api|client.cleancall-opt-1 + code_api|client.cleancallparams + code_api|client.drreg-cross + code_api|client.drreg-end-restore + code_api|client.drreg-flow + code_api|client.emulation_api_simple + code_api|client.execfault + code_api|client.inline + code_api|client.syscall-mod + code_api|client.truncate + code_api|client.unregister + code_api|sample.inscount + code_api|sample.inscount.cleancall + code_api|sample.inscount.prof-pcs.cleancall + code_api|sample.instrace_simple + code_api|sample.memtrace_simple + code_api|sample.opcodes + PROPERTIES LABELS RUNS_ON_QEMU) + endif () + if (ARM) + set_tests_properties( + code_api|common.allasm_arm + code_api|common.allasm_thumb + code_api|common.broadfun-stress + code_api|linux.bad-signal-stack + code_api|linux.alarm + code_api|linux.fork-sleep + code_api|linux.signal_race + code_api|linux.sigplain000 + code_api|linux.sigplain001 + code_api|linux.sigplain010 + code_api|linux.sigplain011 + code_api|linux.sigplain100 + code_api|linux.sigplain101 + code_api|linux.sigplain110 + code_api|linux.sigplain111 + code_api|security-common.TestAllocWE + PROPERTIES LABELS RUNS_ON_QEMU) + endif () +endif (NOT ANDROID AND AARCHXX)