From a339bfefd86a7a915da09674a0d27dc81a8d6d53 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 18 Sep 2019 10:52:51 -0700 Subject: [PATCH 1/2] generated_dts_*: replace ${ZEPHYR_BASE}/ with ZEPHYR_BASE in headers "generated by" comments in the files build/zephyr/include/generated/ generated_dts_board.conf and generated_dts_board_unfixed.h leak absolute and non-deterministic paths to binding dirs. When printing those, search for variable ${ZEPHYR_BASE} and replace that with a literal "ZEPHYR_BASE" constant. Note: this doesn't deal with dts/bindings out of ZEPHYR_BASE. Signed-off-by: Marc Herbert --- scripts/dts/gen_defines.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/dts/gen_defines.py b/scripts/dts/gen_defines.py index 3350495223d9c6..84cd5de125d27b 100755 --- a/scripts/dts/gen_defines.py +++ b/scripts/dts/gen_defines.py @@ -19,6 +19,8 @@ # edtlib. This will keep this script simple. import argparse +import os +from pathlib import Path import sys import edtlib @@ -40,7 +42,20 @@ def main(): out_comment("Generated by gen_defines.py", blank_before=False) out_comment("DTS input file: " + args.dts, blank_before=False) - out_comment("Directories with bindings: " + ", ".join(args.bindings_dirs), + + zbase = Path(os.environ["ZEPHYR_BASE"]) + shorter_bdirs = {} + for d in args.bindings_dirs: + try: + shorter_bdirs[d] = Path(d).relative_to(zbase) + except ValueError: + # leave as is when outside ZEPHYR_BASE + shorter_bdirs[d] = Path(d) + + out_comment("Directories with bindings: " + + ", ".join([ str("ZEPHYR_BASE" / d) + for d in shorter_bdirs.values() + ]), blank_before=False) active_compats = set() From 531a47f047d6787e3d55c60965d5ef424e7078bd Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 17 Sep 2019 16:34:29 -0700 Subject: [PATCH 2/2] edtlib.py: make EDT._compat2binding paths relative This stops leaking private and non-deterministic paths in the (many!) comments of include/generated/generated_dts_board.conf and generated_dts_board_unfixed.h Thanks to Ulf.Magnusson@nordicsemi.no for explaining how this could be done. - Sample lines before: # Device tree node: /ioapic@fec00000 # Binding (compatible = intel,ioapic): \ /home/john/zephyrproject/zephyr/dts/bindings/ \ interrupt-controller/intel,ioapic.yaml /* Device tree node: /ioapic@fec00000 Binding (compatible = intel,ioapic): \ /home/john/zephyrproject/zephyr/dts/bindings/ \ interrupt-controller/intel,ioapic.yaml */ - Same lines after: # Device tree node: /ioapic@fec00000 # DTS Binding (compatible = intel,ioapic): \ interrupt-controller/intel,ioapic.yaml /* Device tree node: /ioapic@fec00000 */ DTS Binding (compatible = intel,ioapic): \ interrupt-controller/intel,ioapic.yaml */ Signed-off-by: Marc Herbert --- scripts/dts/edtlib.py | 31 ++++++++++++++++++++----------- scripts/dts/gen_defines.py | 2 +- scripts/dts/testedtlib.py | 32 ++++++++++++++++---------------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/scripts/dts/edtlib.py b/scripts/dts/edtlib.py index 93f79f98302229..b83c91d536e252 100644 --- a/scripts/dts/edtlib.py +++ b/scripts/dts/edtlib.py @@ -145,7 +145,8 @@ def _init_compat2binding(self, bindings_dirs): # Creates self._compat2binding. This is a dictionary that maps # (, ) tuples (both strings) to (, ) # tuples. is the binding in parsed PyYAML format, and - # the path to the binding (nice for binding-related error messages). + # the relative path to the binding (nice for binding-related + # error messages). # # For example, self._compat2binding["company,dev", "can"] contains the # binding/path for the 'company,dev' device, when it appears on the CAN @@ -169,7 +170,15 @@ def _init_compat2binding(self, bindings_dirs): "|".join(re.escape(compat) for compat in dt_compats) ).search - self._binding_paths = _binding_paths(bindings_dirs) + self._binding_paths = [] + # Relative not to leak private and non-deterministic paths + relpaths = {} + + for bdir in bindings_dirs: + for relp in _binding_rel_yamls(bdir): + abs_yaml_path = os.path.join(bdir, relp) + self._binding_paths.append(abs_yaml_path) + relpaths[abs_yaml_path] = relp self._compat2binding = {} for binding_path in self._binding_paths: @@ -212,7 +221,7 @@ def _init_compat2binding(self, bindings_dirs): _check_binding(binding, binding_path) self._compat2binding[binding_compat, _binding_bus(binding)] = \ - (binding, binding_path) + (binding, relpaths[binding_path]) def _merge_included_bindings(self, binding, binding_path): # Merges any bindings listed in the 'include:' section of 'binding' @@ -1292,17 +1301,17 @@ def _dt_compats(dt): for compat in node.props["compatible"].to_strings()} -def _binding_paths(bindings_dirs): - # Returns a list with the paths to all bindings (.yaml files) in - # 'bindings_dirs' +def _binding_rel_yamls(bindings_dir): + # Returns a list of all the relative paths to all .yaml files in one + # 'bindings_dir' binding_paths = [] - for bindings_dir in bindings_dirs: - for root, _, filenames in os.walk(bindings_dir): - for filename in filenames: - if filename.endswith(".yaml"): - binding_paths.append(os.path.join(root, filename)) + for subdir, _, filenames in os.walk(bindings_dir): + for filename in filenames: + if filename.endswith(".yaml"): + reldir = os.path.relpath(subdir, bindings_dir) + binding_paths.append(os.path.join(reldir, filename)) return binding_paths diff --git a/scripts/dts/gen_defines.py b/scripts/dts/gen_defines.py index 84cd5de125d27b..73ae05e85071bf 100755 --- a/scripts/dts/gen_defines.py +++ b/scripts/dts/gen_defines.py @@ -68,7 +68,7 @@ def main(): continue out_comment("Device tree node: " + dev.path) - out_comment("Binding (compatible = {}): {}".format( + out_comment("DTS Binding (compatible = {}): {}".format( dev.matching_compat, dev.binding_path), blank_before=False) out_comment("Binding description: " + dev.description, diff --git a/scripts/dts/testedtlib.py b/scripts/dts/testedtlib.py index 04de564e6a1454..149716ea2c7e64 100755 --- a/scripts/dts/testedtlib.py +++ b/scripts/dts/testedtlib.py @@ -41,47 +41,47 @@ def verify_streq(actual, expected): # verify_streq(edt.get_dev("/interrupt-parent-test/node").interrupts, - "[, specifier: {'one': 1, 'two': 2, 'three': 3}>, , specifier: {'one': 4, 'two': 5, 'three': 6}>]") + "[, specifier: {'one': 1, 'two': 2, 'three': 3}>, , specifier: {'one': 4, 'two': 5, 'three': 6}>]") verify_streq(edt.get_dev("/interrupts-extended-test/node").interrupts, - "[, specifier: {'one': 1}>, , specifier: {'one': 2, 'two': 3}>, , specifier: {'one': 4, 'two': 5, 'three': 6}>]") + "[, specifier: {'one': 1}>, , specifier: {'one': 2, 'two': 3}>, , specifier: {'one': 4, 'two': 5, 'three': 6}>]") verify_streq(edt.get_dev("/interrupt-map-test/node@0").interrupts, - "[, specifier: {'one': 0}>, , specifier: {'one': 0, 'two': 1}>, , specifier: {'one': 0, 'two': 0, 'three': 2}>]") + "[, specifier: {'one': 0}>, , specifier: {'one': 0, 'two': 1}>, , specifier: {'one': 0, 'two': 0, 'three': 2}>]") verify_streq(edt.get_dev("/interrupt-map-test/node@1").interrupts, - "[, specifier: {'one': 3}>, , specifier: {'one': 0, 'two': 4}>, , specifier: {'one': 0, 'two': 0, 'three': 5}>]") + "[, specifier: {'one': 3}>, , specifier: {'one': 0, 'two': 4}>, , specifier: {'one': 0, 'two': 0, 'three': 5}>]") verify_streq(edt.get_dev("/interrupt-map-bitops-test/node@70000000E").interrupts, - "[, specifier: {'one': 3, 'two': 2}>]") + "[, specifier: {'one': 3, 'two': 2}>]") # # Test GPIOs # verify_streq(edt.get_dev("/gpio-test/node").gpios, - "{'': [, specifier: {'one': 1}>, , specifier: {'one': 2, 'two': 3}>], 'foo': [, specifier: {'one': 4, 'two': 5}>], 'bar': [, specifier: {'one': 6, 'two': 7}>]}") + "{'': [, specifier: {'one': 1}>, , specifier: {'one': 2, 'two': 3}>], 'foo': [, specifier: {'one': 4, 'two': 5}>], 'bar': [, specifier: {'one': 6, 'two': 7}>]}") # # Test clocks # verify_streq(edt.get_dev("/clock-test/node").clocks, - "[, specifier: {}>, , specifier: {'one': 1}>, , specifier: {'one': 1, 'two': 2}>]") + "[, specifier: {}>, , specifier: {'one': 1}>, , specifier: {'one': 1, 'two': 2}>]") # # Test PWMs # verify_streq(edt.get_dev("/pwm-test/node").pwms, - "[, specifier: {}>, , specifier: {'one': 1}>]") + "[, specifier: {}>, , specifier: {'one': 1}>]") # # Test IO channels # verify_streq(edt.get_dev("/io-channel-test/node").iochannels, - "[, specifier: {'one': 1}>]") + "[, specifier: {'one': 1}>]") # # Test 'reg' @@ -131,10 +131,10 @@ def verify_streq(actual, expected): # verify_streq(edt.get_dev("/buses/foo-bus/node").binding_path, - "test-bindings/device-on-foo-bus.yaml") + "./device-on-foo-bus.yaml") verify_streq(edt.get_dev("/buses/bar-bus/node").binding_path, - "test-bindings/device-on-bar-bus.yaml") + "./device-on-bar-bus.yaml") # # Test 'child-binding:' @@ -144,15 +144,15 @@ def verify_streq(actual, expected): child2 = edt.get_dev("/child-binding/child-2") grandchild = edt.get_dev("/child-binding/child-1/grandchild") - verify_streq(child1.binding_path, "test-bindings/child-binding.yaml") + verify_streq(child1.binding_path, "./child-binding.yaml") verify_streq(child1.description, "child node") verify_streq(child1.props, "{'child-prop': }") - verify_streq(child2.binding_path, "test-bindings/child-binding.yaml") + verify_streq(child2.binding_path, "./child-binding.yaml") verify_streq(child2.description, "child node") verify_streq(child2.props, "{'child-prop': }") - verify_streq(grandchild.binding_path, "test-bindings/child-binding.yaml") + verify_streq(grandchild.binding_path, "./child-binding.yaml") verify_streq(grandchild.description, "grandchild node") verify_streq(grandchild.props, "{'grandchild-prop': }") @@ -184,10 +184,10 @@ def verify_streq(actual, expected): edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"]) verify_streq(edt.get_dev("/in-dir-1").binding_path, - "test-bindings/multidir.yaml") + "./multidir.yaml") verify_streq(edt.get_dev("/in-dir-2").binding_path, - "test-bindings-2/multidir.yaml") + "./multidir.yaml") print("all tests passed")