cmake: rework of host tools and generic toolchain handling

Follow-up: #41301

This commit is a rework and cleanup of the tools handling in Zephyr
CMake build system.

Instead of directly loading code a CMake modules for tool lookup, the
host tools now follows the CMake `find_package()` pattern for finding
programs / tools in module mode.

This makes it more clear which modules are responsible for finding tools
and which modules provides build integration / features.

The following tools can now be found using `find_package()`:
- Zephyr-sdk        : find_package(Zephyr-sdk <version>)
- Generic host tools: find_package(HostTools)

This further allows us to decouple the `verify-toolchain` CMake script
part required by `twister` into a tool lookup module and a dedicated
CMake script which utilizes the lookup module.

Signed-off-by: Torsten Rasmussen <[email protected]>

cmake/modules/FindHostTools.cmake

@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) 2022, Nordic Semiconductor ASA
+
+# FindHostTools module for locating a set of tools to use on the host for
+# Zephyr development.
+#
+# This module will lookup the following tools for Zephyr development:
+# +---------------------------------------------------------------+
+# | Tool               | Required |  Notes:                       |
+# +---------------------------------------------------------------+
+# | Generic C-compiler | Yes      |  Pre-processing of devicetree |
+# | Zephyr-sdk         |          |                               |
+# | gperf              |          |                               |
+# | openocd            |          |                               |
+# | bossac             |          |                               |
+# | imgtool            |          |                               |
+# +---------------------------------------------------------------+
+#
+# The module defines the following variables:
+#
+# 'CMAKE_C_COMPILER'
+# Path to C compiler.
+# Set to 'CMAKE_C_COMPILER-NOTFOUND' if no C compiler was found.
+#
+# 'GPERF'
+# Path to gperf.
+# Set to 'GPERF-NOTFOUND' if gperf was not found.
+#
+# 'OPENOCD'
+# Path to openocd.
+# Set to 'OPENOCD-NOTFOUND' if openocd was not found.
+#
+# 'BOSSAC'
+# Path to bossac.
+# Set to 'BOSSAC-NOTFOUND' if bossac was not found.
+#
+# 'IMGTOOL'
+# Path to imgtool.
+# Set to 'IMGTOOL-NOTFOUND' if imgtool was not found.
+#
+# 'HostTools_FOUND', 'HOSTTOOLS_FOUND'
+# True if all required host tools were found.
+
+include(extensions)
+
+if(HostTools_FOUND)
+  return()
+endif()
+
+# gperf is an optional dependency
+find_program(GPERF gperf)
+
+# openocd is an optional dependency
+find_program(OPENOCD openocd)
+
+# bossac is an optional dependency
+find_program(BOSSAC bossac)
+
+# imgtool is an optional dependency (the build may also fall back to scripts/imgtool.py
+# in the mcuboot repository if that's present in some cases)
+find_program(IMGTOOL imgtool)
+
+# Keep XCC_USE_CLANG behaviour for a while.
+if ("${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "xcc"
+    AND "$ENV{XCC_USE_CLANG}" STREQUAL "1")
+  set(ZEPHYR_TOOLCHAIN_VARIANT xcc-clang)
+  message(STATUS "XCC_USE_CLANG is deprecated. Please set ZEPHYR_TOOLCHAIN_VARIANT to 'xcc-clang'")
+endif()
+
+if(NOT ZEPHYR_TOOLCHAIN_VARIANT AND
+   (CROSS_COMPILE OR (DEFINED ENV{CROSS_COMPILE})))
+    set(ZEPHYR_TOOLCHAIN_VARIANT cross-compile)
+endif()
+
+find_package(Zephyr-sdk 0.15)
+
+# Pick host system's toolchain if we are targeting posix
+if("${ARCH}" STREQUAL "posix")
+  if(NOT "${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "llvm")
+    set(ZEPHYR_TOOLCHAIN_VARIANT "host")
+  endif()
+endif()
+
+# Prevent CMake from testing the toolchain
+set(CMAKE_C_COMPILER_FORCED   1)
+set(CMAKE_CXX_COMPILER_FORCED 1)
+
+if(NOT TOOLCHAIN_ROOT)
+  if(DEFINED ENV{TOOLCHAIN_ROOT})
+    # Support for out-of-tree toolchain
+    set(TOOLCHAIN_ROOT $ENV{TOOLCHAIN_ROOT})
+  else()
+    # Default toolchain cmake file
+    set(TOOLCHAIN_ROOT ${ZEPHYR_BASE})
+  endif()
+endif()
+zephyr_file(APPLICATION_ROOT TOOLCHAIN_ROOT)
+
+# Host-tools don't unconditionally set TOOLCHAIN_HOME anymore,
+# but in case Zephyr's SDK toolchain is used, set TOOLCHAIN_HOME
+if("${ZEPHYR_TOOLCHAIN_VARIANT}" STREQUAL "zephyr")
+  set(TOOLCHAIN_HOME ${HOST_TOOLS_HOME})
+endif()
+
+set(TOOLCHAIN_ROOT ${TOOLCHAIN_ROOT} CACHE STRING "Zephyr toolchain root" FORCE)
+assert(TOOLCHAIN_ROOT "Zephyr toolchain root path invalid: please set the TOOLCHAIN_ROOT-variable")
+
+# Set cached ZEPHYR_TOOLCHAIN_VARIANT.
+set(ZEPHYR_TOOLCHAIN_VARIANT ${ZEPHYR_TOOLCHAIN_VARIANT} CACHE STRING "Zephyr toolchain variant")
+
+# Configure the toolchain based on what SDK/toolchain is in use.
+include(${TOOLCHAIN_ROOT}/cmake/toolchain/${ZEPHYR_TOOLCHAIN_VARIANT}/generic.cmake)
+
+# Configure the toolchain based on what toolchain technology is used
+# (gcc, host-gcc etc.)
+include(${TOOLCHAIN_ROOT}/cmake/compiler/${COMPILER}/generic.cmake OPTIONAL)
+include(${TOOLCHAIN_ROOT}/cmake/linker/${LINKER}/generic.cmake OPTIONAL)
+include(${TOOLCHAIN_ROOT}/cmake/bintools/${BINTOOLS}/generic.cmake OPTIONAL)
+
+# Optional folder for toolchains with may provide a Kconfig file for capabilities settings.
+set_ifndef(TOOLCHAIN_KCONFIG_DIR ${TOOLCHAIN_ROOT}/cmake/toolchain/${ZEPHYR_TOOLCHAIN_VARIANT})
+
+set(HostTools_FOUND TRUE)
+set(HOSTTOOLS_FOUND TRUE)

cmake/modules/FindZephyr-sdk.cmake

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) 2022, Nordic Semiconductor ASA
+
+# FindZephyr-sdk module for supporting module search mode of Zephyr SDK.
+#
+# Its purpose is to allow the find_package basic signature mode to lookup Zephyr
+# SDK and based on user / environment settings of selected toolchain decide if
+# the Zephyr SDK CMake package should be loaded.
+#
+# It extends the Zephyr-sdk CMake package by providing more flexibility in when
+# the Zephyr SDK is loaded and loads additional host tools from the Zephyr SDK.
+#
+# The module defines the following variables:
+#
+# 'ZEPHYR_SDK_INSTALL_DIR'
+# Install location of the Zephyr SDK
+#
+# 'ZEPHYR_TOOLCHAIN_VARIANT'
+# Zephyr toolchain variant to use if not defined already.
+#
+# 'Zephyr-sdk_FOUND'
+# True if the Zephyr SDK was found.
+
+# Set internal variables if set in environment.
+if(NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT)
+  set(ZEPHYR_TOOLCHAIN_VARIANT $ENV{ZEPHYR_TOOLCHAIN_VARIANT})
+endif()
+
+if(NOT DEFINED ZEPHYR_SDK_INSTALL_DIR)
+  set(ZEPHYR_SDK_INSTALL_DIR $ENV{ZEPHYR_SDK_INSTALL_DIR})
+endif()
+
+# Load Zephyr SDK Toolchain.
+# There are three scenarios where Zephyr SDK should be looked up:
+# 1) Zephyr specified as toolchain (ZEPHYR_SDK_INSTALL_DIR still used if defined)
+# 2) No toolchain specified == Default to Zephyr toolchain
+# Until we completely deprecate it
+if(("zephyr" STREQUAL ${ZEPHYR_TOOLCHAIN_VARIANT}) OR
+   (NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT) OR
+   (DEFINED ZEPHYR_SDK_INSTALL_DIR))
+
+  # No toolchain was specified, so inform user that we will be searching.
+  if (NOT DEFINED ZEPHYR_SDK_INSTALL_DIR AND
+      NOT DEFINED ZEPHYR_TOOLCHAIN_VARIANT)
+    message(STATUS "ZEPHYR_TOOLCHAIN_VARIANT not set, trying to locate Zephyr SDK")
+  endif()
+
+  # This ensure packages are sorted in descending order.
+  SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT ${CMAKE_FIND_PACKAGE_SORT_DIRECTION})
+  SET(CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT ${CMAKE_FIND_PACKAGE_SORT_ORDER})
+  SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC)
+  SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL)
+
+  if(DEFINED ZEPHYR_SDK_INSTALL_DIR)
+    # The Zephyr SDK will automatically set the toolchain variant.
+    # To support Zephyr SDK tools (DTC, and other tools) with 3rd party toolchains
+    # then we keep track of current toolchain variant.
+    set(ZEPHYR_CURRENT_TOOLCHAIN_VARIANT ${ZEPHYR_TOOLCHAIN_VARIANT})
+    find_package(Zephyr-sdk ${Zephyr-sdk_FIND_VERSION}
+                 REQUIRED QUIET CONFIG HINTS ${ZEPHYR_SDK_INSTALL_DIR}
+    )
+    if(DEFINED ZEPHYR_CURRENT_TOOLCHAIN_VARIANT)
+      set(ZEPHYR_TOOLCHAIN_VARIANT ${ZEPHYR_CURRENT_TOOLCHAIN_VARIANT})
+    endif()
+  else()
+    find_package(Zephyr-sdk ${Zephyr-sdk_FIND_VERSION} REQUIRED QUIET CONFIG PATHS
+                 /usr
+                 /usr/local
+                 /opt
+                 $ENV{HOME}
+                 $ENV{HOME}/.local
+                 $ENV{HOME}/.local/opt
+                 $ENV{HOME}/bin)
+  endif()
+
+  SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION ${CMAKE_FIND_PACKAGE_SORT_DIRECTION_CURRENT})
+  SET(CMAKE_FIND_PACKAGE_SORT_ORDER ${CMAKE_FIND_PACKAGE_SORT_ORDER_CURRENT})
+endif()
+
+if(DEFINED ZEPHYR_SDK_INSTALL_DIR)
+  # Cache the Zephyr SDK install dir.
+  set(ZEPHYR_SDK_INSTALL_DIR ${ZEPHYR_SDK_INSTALL_DIR} CACHE PATH "Zephyr SDK install directory")
+endif()
+
+if(Zephyr-sdk_FOUND)
+  include(${ZEPHYR_SDK_INSTALL_DIR}/cmake/zephyr/host-tools.cmake)
+
+  message(STATUS "Found host-tools: zephyr ${SDK_VERSION} (${ZEPHYR_SDK_INSTALL_DIR})")
+endif()

cmake/modules/dts.cmake

@@ -5,7 +5,7 @@ include_guard(GLOBAL)
 include(extensions)
 include(python)
 include(boards)
-include(generic_toolchain)
+find_package(HostTools)
 find_package(Dtc 1.4.6)
 
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/include/generated)