CMake refactoring for system DT

cmake/modules/dts.cmake

@@ -5,6 +5,7 @@ include_guard(GLOBAL)
 include(extensions)
 include(python)
 include(boards)
+include(pre_dt)
 find_package(HostTools)
 find_package(Dtc 1.4.6)
 
@@ -38,14 +39,53 @@ find_package(Dtc 1.4.6)
 # files in scripts/dts to make all this work. We also optionally will
 # run the dtc tool if it is found, in order to catch any additional
 # warnings or errors it generates.
-
-# We will place some generated include files in here.
-set(BINARY_DIR_INCLUDE           ${PROJECT_BINARY_DIR}/include)
-set(BINARY_DIR_INCLUDE_GENERATED ${BINARY_DIR_INCLUDE}/generated)
-# Unconditionally create it, even if we don't have DTS support. This
-# is a historical artifact, and users expect this directory to exist
-# to put their own generated content inside.
-file(MAKE_DIRECTORY ${BINARY_DIR_INCLUDE_GENERATED})
+#
+# Outcome:
+#
+# 1. The following has happened:
+#
+#    - The pre_dt module has been included; refer to its outcome
+#      section for more information on the consequences
+#    - DTS_SOURCE: set to the path to the devicetree file which
+#      was used, if one was provided or found
+#    - ${BINARY_DIR_INCLUDE_GENERATED}/devicetree_generated.h exists
+#
+# 2. The following has happened if a devicetree was found and
+#    no errors occurred:
+#
+#    - CACHED_DTS_ROOT_BINDINGS is set in the cache to the
+#      value of DTS_ROOT_BINDINGS
+#    - DTS_ROOT_BINDINGS is set to a ;-list of locations where DT
+#      bindings were found
+#    - ${PROJECT_BINARY_DIR}/zephyr.dts exists
+#    - ${PROJECT_BINARY_DIR}/edt.pickle exists
+#    - ${KCONFIG_BINARY_DIR}/Kconfig.dts exists
+#    - the build system will be regenerated if any devicetree files
+#      used in this build change, including transitive includes
+#    - the devicetree extensions in the extensions.cmake module
+#      will be ready for use in other CMake list files that run
+#      after this module
+#
+# Required variables:
+# - BINARY_DIR_INCLUDE_GENERATED: where to put generated include files
+# - KCONFIG_BINARY_DIR: where to put generated Kconfig files
+#
+# Optional variables:
+# - BOARD: board name to use when looking for DTS_SOURCE
+# - BOARD_DIR: board directory to use when looking for DTS_SOURCE
+# - BOARD_REVISION_STRING: used when looking for a board revision's
+#   devicetree overlay file in BOARD_DIR
+# - EXTRA_DTC_FLAGS: list of extra command line options to pass to
+#   dtc when using it to check for additional errors and warnings;
+#   invalid flags are automatically filtered out of the list
+# - DTS_EXTRA_CPPFLAGS: extra command line options to pass to the
+#   C preprocessor when generating the devicetree from DTS_SOURCE
+# - DTS_SOURCE: the devicetree source file to use may be pre-set
+#   with this variable; otherwise, it defaults to
+#   ${BOARD_DIR}/${BOARD.dts}
+#
+# Variables set by this module and not mentioned above are for internal
+# use only, and may be removed, renamed, or re-purposed without prior notice.
 
 # The directory containing devicetree related scripts.
 set(DT_SCRIPTS                  ${ZEPHYR_BASE}/scripts/dts)
@@ -79,9 +119,6 @@ set(DTS_CMAKE                   ${PROJECT_BINARY_DIR}/dts.cmake)
 # modules.
 set(VENDOR_PREFIXES             dts/bindings/vendor-prefixes.txt)
 
-# The C preprocessor to use.
-set_ifndef(CMAKE_DTS_PREPROCESSOR ${CMAKE_C_COMPILER})
-
 #
 # Halt execution early if there is no devicetree.
 #
@@ -101,43 +138,6 @@ else()
   return()
 endif()
 
-#
-# Finalize the value of DTS_ROOT, so we know where all our
-# DTS files, bindings, and vendor prefixes are.
-#
-
-# Convert relative paths to absolute paths relative to the application
-# source directory.
-zephyr_file(APPLICATION_ROOT DTS_ROOT)
-
-# DTS_ROOT always includes the application directory, the board
-# directory, shield directories, and ZEPHYR_BASE.
-list(APPEND
-  DTS_ROOT
-  ${APPLICATION_SOURCE_DIR}
-  ${BOARD_DIR}
-  ${SHIELD_DIRS}
-  ${ZEPHYR_BASE}
-  )
-
-# Convert the directories in DTS_ROOT to absolute paths without
-# symlinks.
-#
-# DTS directories can come from multiple places. Some places, like a
-# user's CMakeLists.txt can preserve symbolic links. Others, like
-# scripts/zephyr_module.py --settings-out resolve them.
-unset(real_dts_root)
-foreach(dts_dir ${DTS_ROOT})
-  file(REAL_PATH ${dts_dir} real_dts_dir)
-  list(APPEND real_dts_root ${real_dts_dir})
-endforeach()
-set(DTS_ROOT ${real_dts_root})
-
-# Finally, de-duplicate the list.
-list(REMOVE_DUPLICATES
-  DTS_ROOT
-  )
-
 #
 # Find all the DTS files we need to concatenate and preprocess, as
 # well as all the devicetree bindings and vendor prefixes associated
@@ -150,25 +150,16 @@ set(dts_files
   )
 
 if(DTC_OVERLAY_FILE)
-  # Convert from space-separated files into file list
-  string(CONFIGURE "${DTC_OVERLAY_FILE}" DTC_OVERLAY_FILE_EXPANDED)
-  string(REPLACE " " ";" DTC_OVERLAY_FILE_RAW_LIST "${DTC_OVERLAY_FILE_EXPANDED}")
-  foreach(file ${DTC_OVERLAY_FILE_RAW_LIST})
-    file(TO_CMAKE_PATH "${file}" cmake_path_file)
-    list(APPEND DTC_OVERLAY_FILE_AS_LIST ${cmake_path_file})
-  endforeach()
+  zephyr_list(TRANSFORM DTC_OVERLAY_FILE NORMALIZE_PATHS
+              OUTPUT_VARIABLE DTC_OVERLAY_FILE_AS_LIST)
   list(APPEND
     dts_files
     ${DTC_OVERLAY_FILE_AS_LIST}
     )
 endif()
 
 set(i 0)
-unset(DTC_INCLUDE_FLAG_FOR_DTS)
 foreach(dts_file ${dts_files})
-  list(APPEND DTC_INCLUDE_FLAG_FOR_DTS
-       -include ${dts_file})
-
   if(i EQUAL 0)
     message(STATUS "Found BOARD.dts: ${dts_file}")
   else()
@@ -178,25 +169,8 @@ foreach(dts_file ${dts_files})
   math(EXPR i "${i}+1")
 endforeach()
 
-unset(DTS_ROOT_SYSTEM_INCLUDE_DIRS)
 unset(DTS_ROOT_BINDINGS)
 foreach(dts_root ${DTS_ROOT})
-  foreach(dts_root_path
-      include
-      include/zephyr
-      dts/common
-      dts/${ARCH}
-      dts
-      )
-    get_filename_component(full_path ${dts_root}/${dts_root_path} REALPATH)
-    if(EXISTS ${full_path})
-      list(APPEND
-        DTS_ROOT_SYSTEM_INCLUDE_DIRS
-        -isystem ${full_path}
-        )
-    endif()
-  endforeach()
-
   set(bindings_path ${dts_root}/dts/bindings)
   if(EXISTS ${bindings_path})
     list(APPEND
@@ -225,29 +199,21 @@ set(CACHED_DTS_ROOT_BINDINGS ${DTS_ROOT_BINDINGS} CACHE INTERNAL
 # regeneration of devicetree_generated.h on every configure. How
 # challenging is this? Can we cache the dts dependencies?
 
-# Run the preprocessor on the DTS input files. We are leaving
-# linemarker directives enabled on purpose. This tells dtlib where
-# each line actually came from, which improves error reporting.
-execute_process(
-  COMMAND ${CMAKE_DTS_PREPROCESSOR}
-  -x assembler-with-cpp
-  -nostdinc
-  ${DTS_ROOT_SYSTEM_INCLUDE_DIRS}
-  ${DTC_INCLUDE_FLAG_FOR_DTS}  # include the DTS source and overlays
-  ${NOSYSDEF_CFLAG}
-  -D__DTS__
-  ${DTS_EXTRA_CPPFLAGS}
-  -E   # Stop after preprocessing
-  -MD  # Generate a dependency file as a side-effect
-  -MF ${DTS_DEPS}
-  -o ${DTS_POST_CPP}
-  ${ZEPHYR_BASE}/misc/empty_file.c
+# Run the preprocessor on the DTS input files.
+if(DEFINED CMAKE_DTS_PREPROCESSOR)
+  set(dts_preprocessor ${CMAKE_DTS_PREPROCESSOR})
+else()
+  set(dts_preprocessor ${CMAKE_C_COMPILER})
+endif()
+zephyr_dt_preprocess(
+  CPP ${dts_preprocessor}
+  SOURCE_FILES ${dts_files}
+  OUT_FILE ${DTS_POST_CPP}
+  DEPS_FILE ${DTS_DEPS}
+  EXTRA_CPPFLAGS ${DTS_EXTRA_CPPFLAGS}
+  INCLUDE_DIRECTORIES ${DTS_ROOT_SYSTEM_INCLUDE_DIRS}
   WORKING_DIRECTORY ${APPLICATION_SOURCE_DIR}
-  RESULT_VARIABLE ret
   )
-if(NOT "${ret}" STREQUAL "0")
-  message(FATAL_ERROR "command failed with return code: ${ret}")
-endif()
 
 #
 # Make sure we re-run CMake if any devicetree sources or transitive

cmake/modules/extensions.cmake

@@ -30,6 +30,7 @@ include(CheckCXXCompilerFlag)
 # 4. Devicetree extensions
 # 4.1 dt_*
 # 4.2. *_if_dt_node
+# 4.3  zephyr_dt_*
 # 5. Zephyr linker functions
 # 5.1. zephyr_linker*
 # 6 Function helper macros
@@ -2410,6 +2411,57 @@ function(zephyr_string)
   set(${return_arg} ${work_string} PARENT_SCOPE)
 endfunction()
 
+# Usage:
+#    zephyr_list(TRANSFORM <list> <ACTION>
+#                [OUTPUT_VARIABLE <output variable])
+#
+# Example:
+#
+#    zephyr_list(TRANSFORM my_input_var NORMALIZE_PATHS
+#                OUTPUT_VARIABLE my_input_as_list)
+#
+# Like CMake's list(TRANSFORM ...). This is intended as a placeholder
+# for storing current and future Zephyr-related extensions for list
+# processing.
+#
+# <ACTION>: This currently must be NORMALIZE_PATHS. This action
+#           converts the argument list <list> to a ;-list with
+#           CMake path names, after passing its contents through
+#           a configure_file() transformation. The input list
+#           may be whitespace- or semicolon-separated.
+#
+# OUTPUT_VARIABLE: the result is normally stored in place, but
+#                  an alternative variable to store the result
+#                  can be provided with this.
+function(zephyr_list transform list_var action)
+  # Parse arguments.
+  if(NOT "${transform}" STREQUAL "TRANSFORM")
+    message(FATAL_ERROR "the first argument must be TRANSFORM")
+  endif()
+  if(NOT "${action}" STREQUAL "NORMALIZE_PATHS")
+    message(FATAL_ERROR "the third argument must be NORMALIZE_PATHS")
+  endif()
+  set(single_args OUTPUT_VARIABLE)
+  cmake_parse_arguments(ZEPHYR_LIST "" "${single_args}" "" ${ARGN})
+  if(DEFINED ZEPHYR_LIST_OUTPUT_VARIABLE)
+    set(out_var ${ZEPHYR_LIST_OUTPUT_VARIABLE})
+  else()
+    set(out_var ${list_var})
+  endif()
+  set(input ${${list_var}})
+
+  # Perform the transformation.
+  set(ret)
+  string(CONFIGURE "${input}" input_expanded)
+  string(REPLACE " " ";" input_raw_list "${input_expanded}")
+  foreach(file ${input_raw_list})
+    file(TO_CMAKE_PATH "${file}" cmake_path_file)
+    list(APPEND ret ${cmake_path_file})
+  endforeach()
+
+  set(${out_var} ${ret} PARENT_SCOPE)
+endfunction()
+
 # Usage:
 #   zephyr_get(<variable>)
 #   zephyr_get(<variable> SYSBUILD [LOCAL|GLOBAL])
@@ -3427,7 +3479,108 @@ function(target_sources_if_dt_node path target scope item)
 endfunction()
 
 ########################################################
-# 5. Zephyr linker function
+# 4.3 zephyr_dt_*
+#
+# The following methods are common code for dealing
+# with devicetree related files in CMake.
+#
+# Note that functions related to accessing the
+# *contents* of the devicetree belong in section 4.1.
+# This section is just for DT file processing at
+# configuration time.
+########################################################
+
+# Usage:
+#   zephyr_dt_preprocess(CPP <path>
+#                        SOURCE_FILES <files>
+#                        OUT_FILE <file>
+#                        [DEPS_FILE <file>]
+#                        [EXTRA_CPPFLAGS <flags>]
+#                        [INCLUDE_DIRECTORIES <dirs>]
+#                        [WORKING_DIRECTORY <dir>]
+#
+# Preprocess one or more devicetree source files. The preprocessor
+# symbol __DTS__ will be defined. If the preprocessor command fails, a
+# fatal error occurs. CMAKE_DTS_PREPROCESSOR is used as the
+# preprocessor.
+#
+# Mandatory arguments:
+#
+# CPP <path>: path to C preprocessor
+#
+# SOURCE_FILES <files>: The source files to run the preprocessor on.
+#                       These will, in effect, be concatenated in order
+#                       and used as the preprocessor input.
+#
+# OUT_FILE <file>: Where to store the preprocessor output.
+#
+# Optional arguments:
+#
+# DEPS_FILE <file>: If set, generate a dependency file here.
+#
+# EXTRA_CPPFLAGS <flags>: Additional flags to pass the preprocessor.
+#
+# INCLUDE_DIRECTORIES <dirs>: Additional directories containing #included
+#                             files.
+#
+# WORKING_DIRECTORY <dir>: where to run the preprocessor.
+function(zephyr_dt_preprocess)
+  set(req_single_args "CPP;OUT_FILE")
+  set(single_args "DEPS_FILE;WORKING_DIRECTORY")
+  set(multi_args "SOURCE_FILES;EXTRA_CPPFLAGS;INCLUDE_DIRECTORIES")
+  cmake_parse_arguments(DT_PREPROCESS "" "${req_single_args};${single_args}" "${multi_args}" ${ARGN})
+
+  foreach(arg ${req_single_args} SOURCE_FILES)
+    if(NOT DEFINED DT_PREPROCESS_${arg})
+      message(FATAL_ERROR "dt_preprocess() missing required argument: ${arg}")
+    endif()
+  endforeach()
+
+  set(include_opts)
+  foreach(dir ${DT_PREPROCESS_INCLUDE_DIRECTORIES})
+    list(APPEND include_opts -isystem ${dir})
+  endforeach()
+
+  set(source_opts)
+  foreach(file ${DT_PREPROCESS_SOURCE_FILES})
+    list(APPEND source_opts -include ${file})
+  endforeach()
+
+  set(deps_opts)
+  if(DEFINED DT_PREPROCESS_DEPS_FILE)
+    list(APPEND deps_opts -MD -MF ${DT_PREPROCESS_DEPS_FILE})
+  endif()
+
+  set(workdir_opts)
+  if(DEFINED DT_PREPROCESS_WORKING_DIRECTORY)
+    list(APPEND workdir_opts WORKING_DIRECTORY ${DT_PREPROCESS_WORKING_DIRECTORY})
+  endif()
+
+  # We are leaving linemarker directives enabled on purpose. This tells
+  # dtlib where each line actually came from, which improves error
+  # reporting.
+  set(preprocess_cmd ${DT_PREPROCESS_CPP}
+    -x assembler-with-cpp
+    -nostdinc
+    ${include_opts}
+    ${source_opts}
+    ${NOSYSDEF_CFLAG}
+    -D__DTS__
+    ${DT_PREPROCESS_EXTRA_CPPFLAGS}
+    -E   # Stop after preprocessing
+    ${deps_opts}
+    -o ${DT_PREPROCESS_OUT_FILE}
+    ${ZEPHYR_BASE}/misc/empty_file.c
+    ${workdir_opts})
+
+  execute_process(COMMAND ${preprocess_cmd} RESULT_VARIABLE ret)
+  if(NOT "${ret}" STREQUAL "0")
+    message(FATAL_ERROR "failed to preprocess devicetree files (error code ${ret}): ${DT_PREPROCESS_SOURCE_FILES}")
+  endif()
+endfunction()
+
+########################################################
+# 5. Zephyr linker functions
 ########################################################
 # 5.1. zephyr_linker*
 #