trezor: support v2.5.2+, add more trezor tests, fix chaingen and tests

- passphrase logic: remove backward compatibility for 2.4.3, code cleanup.
- fix LibUSB cmake for static builds on OSX
- tests: all tests now work with passphrase logic enabled. Passphrase test added with different passphrase. no_passphrase test added, Trezor pin test added. Testing wallet opening with correct and incorrect passphrase. Trezor test chain revamp, cleanup. Smaller chain, chain file versioning added.
- tests: Trezor tests support TEST_MINING_ENABLED, TEST_MINING_TIMEOUT env vars to change mining-related tests behaviour.
- tests: fix tx create from sources, input locking. Originally, creating a synthetic transactions with chaingen could create a transaction with outputs that are still locked in the current block, thus failing chain validation by the daemon. Simple unlock check was added. Some buggy tests were fixed as well as new unlock-checking version of tx creation rejected those, fixes are simple - mostly using correct block after a rewind to construct a transaction
- requires protobuf@21 on osx for now (c++14), building with unlinked protobuf: `CMAKE_PREFIX_PATH=$(find /opt/homebrew/Cellar/protobuf@21 -maxdepth 1 -type d -name "21.*" -print -quit) \
 make debug-test-trezor -j8`

.github/workflows/build.yml

@@ -23,6 +23,7 @@ env:
   CCACHE_SETTINGS: |
         ccache --max-size=150M
         ccache --set-config=compression=true
+  USE_DEVICE_TREZOR_MANDATORY: ON
 
 jobs:
   build-macos:
@@ -39,7 +40,9 @@ jobs:
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
     - name: install dependencies
-      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
+      run: |
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf@21 ccache
+        brew link protobuf@21
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
@@ -65,7 +68,12 @@ jobs:
     - uses: msys2/setup-msys2@v2
       with:
         update: true
-        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
+        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
+    - shell: msys2 {0}
+      run: |
+           curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst
+           curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
+           pacman --noconfirm -U mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}

.github/workflows/depends.yml

@@ -18,6 +18,7 @@ env:
   CCACHE_SETTINGS: |
         ccache --max-size=150M
         ccache --set-config=compression=true
+  USE_DEVICE_TREZOR_MANDATORY: ON
 
 jobs:
   build-cross:

README.md

@@ -590,6 +590,16 @@ Using `depends` might also be easier to compile Monero on Windows than using MSY
 
 The produced binaries still link libc dynamically. If the binary is compiled on a current distribution, it might not run on an older distribution with an older installation of libc. Passing `-DBACKCOMPAT=ON` to cmake will make sure that the binary will run on systems having at least libc version 2.17.
 
+### Trezor hardware wallet support
+
+If you have an issue with building Monero with Trezor support, you can disable it by setting `USE_DEVICE_TREZOR=OFF`, e.g., 
+
+```bash
+USE_DEVICE_TREZOR=OFF make release
+```
+
+For more information, please check out Trezor [src/device_trezor/README.md](src/device_trezor/README.md).
+
 ### Gitian builds
 
 See [contrib/gitian/README.md](contrib/gitian/README.md).

cmake/CheckTrezor.cmake

@@ -1,8 +1,27 @@
-OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON)
-OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON)
-OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF)
-OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" OFF)
-OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF)
+# Function for setting default options default values via env vars
+function(_trezor_default_val val_name val_default)
+    if(NOT DEFINED ENV{${val_name}})
+        set(ENV{${val_name}} ${val_default})
+    endif()
+endfunction()
+
+# Define default options via env vars
+_trezor_default_val(USE_DEVICE_TREZOR ON)
+_trezor_default_val(USE_DEVICE_TREZOR_MANDATORY OFF)
+_trezor_default_val(USE_DEVICE_TREZOR_PROTOBUF_TEST ON)
+_trezor_default_val(USE_DEVICE_TREZOR_LIBUSB ON)
+_trezor_default_val(USE_DEVICE_TREZOR_UDP_RELEASE OFF)
+_trezor_default_val(USE_DEVICE_TREZOR_DEBUG OFF)
+_trezor_default_val(TREZOR_DEBUG OFF)
+
+# Main options
+OPTION(USE_DEVICE_TREZOR "Trezor support compilation" $ENV{USE_DEVICE_TREZOR})
+OPTION(USE_DEVICE_TREZOR_MANDATORY "Trezor compilation is mandatory, fail build if Trezor support cannot be compiled" $ENV{USE_DEVICE_TREZOR_MANDATORY})
+OPTION(USE_DEVICE_TREZOR_PROTOBUF_TEST "Trezor Protobuf test" $ENV{USE_DEVICE_TREZOR_PROTOBUF_TEST})
+OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" $ENV{USE_DEVICE_TREZOR_LIBUSB})
+OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" $ENV{USE_DEVICE_TREZOR_UDP_RELEASE})
+OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" $ENV{USE_DEVICE_TREZOR_DEBUG})
+OPTION(TREZOR_DEBUG "Main Trezor debugging switch" $ENV{TREZOR_DEBUG})
 
 # Helper function to fix cmake < 3.6.0 FindProtobuf variables
 function(_trezor_protobuf_fix_vars)
@@ -30,41 +49,70 @@ function(_trezor_protobuf_fix_vars)
     endif()
 endfunction()
 
+macro(trezor_fatal_msg msg)
+    if ($ENV{USE_DEVICE_TREZOR_MANDATORY})
+        message(FATAL_ERROR
+                "${msg}\n"
+                "==========================================================================\n"
+                "[ERROR] To compile without Trezor support, set USE_DEVICE_TREZOR=OFF. "
+                "It is possible both via cmake variable and environment variable, e.g., "
+                "`USE_DEVICE_TREZOR=OFF make release`\n"
+                "For more information, please check src/device_trezor/README.md\n"
+        )
+    else()
+        message(WARNING
+                "${msg}\n"
+                "==========================================================================\n"
+                "[WARNING] Trezor support cannot be compiled! Skipping Trezor compilation. \n"
+                "For more information, please check src/device_trezor/README.md\n")
+        set(USE_DEVICE_TREZOR OFF)
+        return()  # finish this cmake file processing (as this is macro).
+    endif()
+endmacro()
+
 # Use Trezor master switch
 if (USE_DEVICE_TREZOR)
     # Protobuf is required to build protobuf messages for Trezor
     include(FindProtobuf OPTIONAL)
-    find_package(Protobuf)
+
+    FIND_PACKAGE(Protobuf CONFIG)
+    if (NOT Protobuf_FOUND)
+        FIND_PACKAGE(Protobuf)
+    endif()
+
     _trezor_protobuf_fix_vars()
 
-    # Protobuf handling the cache variables set in docker.
-    if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
-        message(STATUS "Could not find Protobuf")
+    # Early fail for optional Trezor support
+    if(${Protobuf_VERSION} GREATER 21)
+        trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.")
+    elseif(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
+        trezor_fatal_msg("Trezor: Could not find Protobuf")
     elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}")
-        message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}")
+        trezor_fatal_msg("Trezor: Protobuf library not found: ${Protobuf_LIBRARY}")
         unset(Protobuf_FOUND)
     elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
-        message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
+        trezor_fatal_msg("Trezor: Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
         unset(Protobuf_FOUND)
     elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}")
-        message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
+        trezor_fatal_msg("Trezor: Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
         unset(Protobuf_FOUND)
     else()
-        message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
+        message(STATUS "Trezor: Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
         set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR})
-        set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
+        set(Protobuf_FOUND 1)  # override found if all required info was provided by variables
     endif()
 
     if(TREZOR_DEBUG)
         set(USE_DEVICE_TREZOR_DEBUG 1)
+        message(STATUS "Trezor: debug build enabled")
     endif()
 
     # Compile debugging support (for tests)
     if (USE_DEVICE_TREZOR_DEBUG)
         add_definitions(-DWITH_TREZOR_DEBUGGING=1)
     endif()
 else()
-    message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR")
+    message(STATUS "Trezor: support disabled by USE_DEVICE_TREZOR")
 endif()
 
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
@@ -85,35 +133,50 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
     endif()
 
     if(NOT TREZOR_PYTHON)
-        message(STATUS "Trezor: Python not found")
+        trezor_fatal_msg("Trezor: Python not found")
     endif()
 endif()
 
 # Protobuf compilation test
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
     execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_CURRENT_LIST_DIR}" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
-        message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}")
-    endif()
-
-    try_compile(Protobuf_COMPILE_TEST_PASSED
-        "${CMAKE_BINARY_DIR}"
-        SOURCES
-        "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc"
-        "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.cpp"
-        CMAKE_FLAGS
-        "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}"
-        "-DCMAKE_CXX_STANDARD=11"
-        LINK_LIBRARIES ${Protobuf_LIBRARY}
-        OUTPUT_VARIABLE OUTPUT
-    )
-    if(NOT Protobuf_COMPILE_TEST_PASSED)
-        message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.")
+        trezor_fatal_msg("Trezor: Protobuf test generation failed: ${OUT} ${ERR}")
+    endif()
+
+    if(ANDROID)
+        set(CMAKE_TRY_COMPILE_LINKER_FLAGS "${CMAKE_TRY_COMPILE_LINKER_FLAGS} -llog")
+        set(CMAKE_TRY_COMPILE_LINK_LIBRARIES "${CMAKE_TRY_COMPILE_LINK_LIBRARIES} log")
+    endif()
+
+    if(USE_DEVICE_TREZOR_PROTOBUF_TEST)
+        # For now, Protobuf v21 is the maximum supported version as v23 requires C++17. TODO: Remove once we move to C++17
+        if(${Protobuf_VERSION} GREATER 21)
+            trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.")
+        endif()
+
+        try_compile(Protobuf_COMPILE_TEST_PASSED
+            "${CMAKE_BINARY_DIR}"
+            SOURCES
+            "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc"
+            "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.cpp"
+            CMAKE_FLAGS
+            CMAKE_EXE_LINKER_FLAGS ${CMAKE_TRY_COMPILE_LINKER_FLAGS}
+            "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}"
+            "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}"
+            LINK_LIBRARIES ${Protobuf_LIBRARY} ${CMAKE_TRY_COMPILE_LINK_LIBRARIES}
+            OUTPUT_VARIABLE OUTPUT
+        )
+        if(NOT Protobuf_COMPILE_TEST_PASSED)
+            trezor_fatal_msg("Trezor: Protobuf Compilation test failed: ${OUTPUT}.")
+        endif()
+    else ()
+        message(STATUS "Trezor: Protobuf Compilation test skipped, build may fail later")
     endif()
 endif()
 
 # Try to build protobuf messages
-if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED)
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
     set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}")
     set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}")
     set(TREZOR_PROTOBUF_PARAMS "")
@@ -123,59 +186,62 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T
 
     execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py ${TREZOR_PROTOBUF_PARAMS} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
-        message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
+        trezor_fatal_msg("Trezor: protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
                 "OUT: ${OUT}, ERR: ${ERR}."
                 "Please read src/device_trezor/trezor/tools/README.md")
-    else()
-        message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"")
-        set(DEVICE_TREZOR_READY 1)
-        add_definitions(-DDEVICE_TREZOR_READY=1)
-        add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
+    endif()
 
-        if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-            add_definitions(-DTREZOR_DEBUG=1)
-        endif()
+    message(STATUS "Trezor: protobuf messages regenerated out: \"${OUT}.\"")
+    set(DEVICE_TREZOR_READY 1)
+    add_definitions(-DDEVICE_TREZOR_READY=1)
+    add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
 
-        if(USE_DEVICE_TREZOR_UDP_RELEASE)
-            add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
-        endif()
+    if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+        add_definitions(-DTREZOR_DEBUG=1)
+    endif()
 
-        if (Protobuf_INCLUDE_DIR)
-            include_directories(${Protobuf_INCLUDE_DIR})
-        endif()
+    if(USE_DEVICE_TREZOR_UDP_RELEASE)
+        message(STATUS "Trezor: UDP transport enabled (emulator)")
+        add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
+    endif()
 
-        # LibUSB support, check for particular version
-        # Include support only if compilation test passes
-        if (USE_DEVICE_TREZOR_LIBUSB)
-            find_package(LibUSB)
-        endif()
+    if (Protobuf_INCLUDE_DIR)
+        include_directories(${Protobuf_INCLUDE_DIR})
+    endif()
 
-        if (LibUSB_COMPILE_TEST_PASSED)
-            add_definitions(-DHAVE_TREZOR_LIBUSB=1)
-            if(LibUSB_INCLUDE_DIRS)
-                include_directories(${LibUSB_INCLUDE_DIRS})
-            endif()
-        endif()
+    # LibUSB support, check for particular version
+    # Include support only if compilation test passes
+    if (USE_DEVICE_TREZOR_LIBUSB)
+        find_package(LibUSB)
+    endif()
 
-        set(TREZOR_LIBUSB_LIBRARIES "")
-        if(LibUSB_COMPILE_TEST_PASSED)
-            list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER})
-            message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+    if (LibUSB_COMPILE_TEST_PASSED)
+        add_definitions(-DHAVE_TREZOR_LIBUSB=1)
+        if(LibUSB_INCLUDE_DIRS)
+            include_directories(${LibUSB_INCLUDE_DIRS})
         endif()
+    endif()
+
+    set(TREZOR_LIBUSB_LIBRARIES "")
+    if(LibUSB_COMPILE_TEST_PASSED)
+        list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER})
+        message(STATUS "Trezor: compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+    elseif(USE_DEVICE_TREZOR_LIBUSB AND NOT ANDROID)
+        trezor_fatal_msg("Trezor: LibUSB not found or test failed, please install libusb-1.0.26")
+    endif()
 
-        if (BUILD_GUI_DEPS)
-            set(TREZOR_DEP_LIBS "")
-            set(TREZOR_DEP_LINKER "")
+    if (BUILD_GUI_DEPS)
+        set(TREZOR_DEP_LIBS "")
+        set(TREZOR_DEP_LINKER "")
 
-            if (Protobuf_LIBRARY)
-                list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY})
-                string(APPEND TREZOR_DEP_LINKER " -lprotobuf")
-            endif()
+        if (Protobuf_LIBRARY)
+            list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY})
+            string(APPEND TREZOR_DEP_LINKER " -lprotobuf")
+        endif()
 
-            if (TREZOR_LIBUSB_LIBRARIES)
-                list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES})
-                string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}")
-            endif()
+        if (TREZOR_LIBUSB_LIBRARIES)
+            list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES})
+            string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}")
         endif()
     endif()
 endif()