Merge pull request #20366 from maribu/makefiles/existing_features

build system: add list of features for documentation and sanity checking

Makefile

@@ -7,6 +7,9 @@ all: welcome
 	@exit 1
 
 doc:
+	@./dist/tools/features_yaml2mx/features_yaml2mx.py \
+		features.yaml \
+		--output-md doc/doxygen/src/feature_list.md
 	"$(MAKE)" -BC doc/doxygen
 
 doc-man:
@@ -33,6 +36,11 @@ distclean: docclean pkg-clean
 print-versions:
 	@./dist/tools/ci/print_toolchain_versions.sh
 
+generate-features:
+	@./dist/tools/features_yaml2mx/features_yaml2mx.py \
+		features.yaml \
+		--output-makefile makefiles/features_existing.inc.mk
+
 include makefiles/boards.inc.mk
 include makefiles/app_dirs.inc.mk
 
@@ -53,7 +61,7 @@ welcome:
 	@echo "    https://forum.riot-os.org"
 	@echo ""
 	@echo "Available targets for the RIOT base directory include:"
-	@echo " generate-{board,driver,example,module,pkg,test}"
+	@echo " generate-{board,driver,example,module,pkg,test,features}"
 	@echo " info-{applications,boards,emulated-boards} info-applications-supported-boards"
 	@echo " print-versions"
 	@echo " clean distclean pkg-clean"

boards/esp32-olimex-evb/Kconfig

@@ -20,7 +20,6 @@ config BOARD_ESP32_OLIMEX_EVB
     select HAS_PERIPH_SDMMC
     select HAS_PERIPH_SPI
     select HAS_PERIPH_CAN
-    select HAS_PERIPH_IR
 
     select HAVE_MTD_SDMMC_DEFAULT
 

boards/esp32-olimex-evb/Makefile.features

@@ -21,6 +21,5 @@ FEATURES_CONFLICT_MSG += "SD/MMC and SPI cannot be used at the same time on this
 # unique features of the board
 FEATURES_PROVIDED += esp_eth        # Ethernet MAC (EMAC)
 FEATURES_PROVIDED += periph_can     # CAN peripheral interface
-FEATURES_PROVIDED += periph_ir      # IR peripheral interface
 
 FEATURES_PROVIDED += arduino_pins

dist/tools/ci/check_features_existing_inc_mk_is_up_to_date.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# This tools checks if the makefiles/features_existing.inc.mk is up to date
+hash_feature_list_pre="$(sha256sum "$(dirname "$0")/../../../makefiles/features_existing.inc.mk")"
+make --silent -C "$(dirname "$0")"/../../.. generate-features || exit 1
+hash_feature_list_post="$(sha256sum "$(dirname "$0")/../../../makefiles/features_existing.inc.mk")"
+if [ "$hash_feature_list_pre" != "$hash_feature_list_post" ]; then
+    echo "Forgot to run make generate-features after updating features.yaml!"
+    exit 1
+fi
+
+exit 0

dist/tools/ci/static_tests.sh

@@ -114,6 +114,7 @@ export BASE_BRANCH="${CI_BASE_BRANCH}"
 run ./dist/tools/whitespacecheck/check.sh "${BASE_BRANCH}"
 DIFFFILTER="MR" ERROR_EXIT_CODE=0 run ./dist/tools/licenses/check.sh
 DIFFFILTER="AC" run ./dist/tools/licenses/check.sh
+run ./dist/tools/ci/check_features_existing_inc_mk_is_up_to_date.sh
 run ./dist/tools/doccheck/check.sh
 run ./dist/tools/externc/check.sh
 # broken configuration produces many false positives

dist/tools/features_yaml2mx/README.md

@@ -0,0 +1,12 @@
+Tool to export features described in a YAML to formats usable by our build system
+=================================================================================
+
+The python script in this folder converts features specified in a YAML file to
+a Makefile for use to check provided and requested features against, as well
+as a markdown file for consumption by Doxygen to document the features.
+
+The YAML file containing the features is `features.yaml` in the root of the
+repo. Its syntax is documented in a comment on top of said file.
+
+This script should be invoked by running `make generate-features` in the root
+of the repository.

dist/tools/features_yaml2mx/features_yaml2mx.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+Command line utility generate trivial Makefile listing all existing features in
+RIOT and a matching documentation in Markdown format from single YAML file.
+"""
+import argparse
+import yaml
+
+
+def collect_features(parsed):
+    """
+    Collect all features from a parsed YAML file
+
+    :param parsed: Parsed YAML file
+    :type parsed: dict
+    :return: list of features in no particular, possible unstable, order
+    :rtype: list
+    """
+    result = []
+    for group in parsed.get("groups", []):
+        result += collect_features(group)
+    for feature in parsed.get("features", []):
+        result.append(feature["name"])
+    return result
+
+
+def write_makefile(outfile, yaml_path, parsed):
+    """
+    Extract the list of features from the given parsed YAML file and writes
+    them into file at the given path in Makefile syntax, e.g.
+
+        FEATURES_EXISTING := \
+            feat_a \
+            feat_b \
+            feat_c \
+            #
+
+    :param outfile: path to the Makefile to write the features to
+    :type outfile: str
+    :param yaml_path:   Path to the source YAML file
+    :type yaml_path:    str
+    :param parsed: the parsed YAML file
+    :type parsed: dict
+    """
+    outfile.write(f"""\
+# WARNING: This has been auto-generated from {yaml_path}.
+#          Do not edit this by hand, but update {yaml_path} instead.
+#          Finally, run `make generate-features` in the root of the RIOT repo.
+""")
+    outfile.write("FEATURES_EXISTING := \\\n")
+    for feature in sorted(collect_features(parsed)):
+        outfile.write(f"    {feature} \\\n")
+    outfile.write("    #\n")
+    outfile.flush()
+
+
+def write_md_section(outfile, group, level):
+    """
+    Write a section documenting certain features to the given file in markdown
+    format.
+
+    :param outfile: The file to write the section to
+    :type outfile:  file
+    :param group:   The group content (e.g. a subtree from the parsed YAML)
+    :type group:    dict
+    :param level:   The current section level (e.g. 1=section, 2=subsection)
+    :type level:    int
+    """
+    title = group.get("title")
+    outfile.write("#" * level + f" {title}" if title else "" + "\n")
+    if "help" in group:
+        outfile.write("\n")
+        outfile.write(group["help"])
+        outfile.write("\n")
+
+    if "features" in group:
+        outfile.write("\n")
+        outfile.write("""\
+| Feature                           | Description                                                                   |
+|:--------------------------------- |:----------------------------------------------------------------------------- |
+""")
+
+        for feature in group["features"]:
+            name = f"`{feature['name']}`"
+            description = feature['help'].strip().replace("\n", " ")
+            outfile.write(f"| {name:<33} | {description:<77} |\n")
+
+    for group in group.get('groups', []):
+        outfile.write("\n")
+        write_md_section(outfile, group, level + 1)
+
+
+def write_mdfile(outfile, yaml_path, parsed):
+    """
+    Write the given contents from the parsed YAML file as markdown
+    documentation to the given file
+
+    :param outfile:     The file to write the documentation to
+    :type outfile:      file
+    :param yaml_path:   Path to the source YAML file
+    :type yaml_path:    str
+    :param parsed:      The parsed YAML file contents
+    :type parsed:       dict
+    """
+    outfile.write(f"""\
+# List of Features (Features as Build System Enties)
+<!--
+WARNING: This has been auto-generated from {yaml_path}.
+         Do not edit this by hand, but update {yaml_path} instead.
+         Finally, run `make generate-features` in the root of the RIOT repo.
+-->
+
+[TOC]
+
+""")
+    write_md_section(outfile, parsed, 0)
+
+
+def convert_features(yaml_file, mk_file, md_file):
+    """
+    Convert the YAML file identified by the given path to a Makefile and
+    to a markdown file, if their paths are given.
+
+    :param yaml_file:   Path to the YAML file to read
+    :type yaml_file:    str
+    :param mk_file:     Path to the Makefile to write the features to or None
+                        for not writing the Makefile
+    :type mk_file:      str or None
+    :param md_file:     Path to the markdown file to write the doc to or None
+                        for not writing the doc
+    :type md_file:      str or None
+    """
+    with open(yaml_file, 'rb') as file:
+        parsed = yaml.safe_load(file)
+
+    if mk_file is not None:
+        with open(mk_file, 'w', encoding="utf-8") as file:
+            write_makefile(file, yaml_file, parsed)
+
+    if md_file is not None:
+        with open(md_file, 'w', encoding="utf-8") as file:
+            write_mdfile(file, yaml_file, parsed)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+            description='Generate documentation for features in markdown ' +
+                        'format and a Makefile listing existing features'
+            )
+    parser.add_argument('INPUT', type=str, default=None,
+                        help="Input file in YAML format")
+    parser.add_argument('--output-md', type=str, default=None,
+                        help="Output file to write the markdown " +
+                             "documentation to (default: no documentation")
+    parser.add_argument('--output-makefile', type=str, default=None,
+                        help="Output file to write the makefile to " +
+                             "(default: no makefile generated)")
+
+    args = parser.parse_args()
+
+    convert_features(args.INPUT, args.output_makefile, args.output_md)

dist/tools/features_yaml2mx/schema.cddl

@@ -0,0 +1,16 @@
+root = group-toplevel
+
+group-toplevel = group .within { "groups" => any }
+group-nested = group .within { "title" => tstr, any => any }
+
+group = {
+    ? "title" => tstr,
+    ? "help" => tstr,
+    ? "features" => [ + feature ],
+    ? "groups" => [ + group-nested ],
+    }
+
+feature = {
+    "name" => tstr,
+    ? "help" => tstr,
+}

doc/doxygen/riot.doxyfile

@@ -769,6 +769,7 @@ INPUT                  = ../../doc.txt \
                          src/build-in-docker.md \
                          ../../tests/README.md \
                          src/build-system-basics.md \
+                         src/feature_list.md \
                          src/kconfig/kconfig.md \
                          src/using-cpp.md \
                          src/using-rust.md \

doc/doxygen/src/.gitignore

@@ -0,0 +1 @@
+/feature_list.md