Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Relative and deterministic DT comments #19333

Closed
wants to merge 2 commits into from
Closed

Relative and deterministic DT comments #19333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a339bfe
generated_dts_*: replace ${ZEPHYR_BASE}/ with ZEPHYR_BASE in headers
marc-hb Sep 18, 2019
531a47f
edtlib.py: make EDT._compat2binding paths relative
marc-hb Sep 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 20 additions & 11 deletions scripts/dts/edtlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ def _init_compat2binding(self, bindings_dirs):
# Creates self._compat2binding. This is a dictionary that maps
# (<compatible>, <bus>) tuples (both strings) to (<binding>, <path>)
# tuples. <binding> is the binding in parsed PyYAML format, and <path>
# the path to the binding (nice for binding-related error messages).
# the relative path to the binding (nice for binding-related
# error messages).
#
# For example, self._compat2binding["company,dev", "can"] contains the
# binding/path for the 'company,dev' device, when it appears on the CAN
Expand All @@ -169,7 +170,15 @@ def _init_compat2binding(self, bindings_dirs):
"|".join(re.escape(compat) for compat in dt_compats)
).search

self._binding_paths = _binding_paths(bindings_dirs)
self._binding_paths = []
# Relative not to leak private and non-deterministic paths
relpaths = {}

for bdir in bindings_dirs:
for relp in _binding_rel_yamls(bdir):
abs_yaml_path = os.path.join(bdir, relp)
self._binding_paths.append(abs_yaml_path)
relpaths[abs_yaml_path] = relp
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tinkered a bit and came up with a dict-less version.

        # Used by 'include:'
        self._binding_paths = []
        # Stores paths relative to the binding directory. Used for output, to
        # make it deterministic.
        relpaths = []

        for bdir in bindings_dirs:
            for abspath in _binding_paths(bdir):
                self._binding_paths.append(abspath)
                relpaths.append(os.path.relpath(abspath, bdir))

        self._compat2binding = {}
        for binding_abspath, binding_relpath in zip(self._binding_paths, relpaths):
            with open(binding_abspath, encoding="utf-8") as f:
                contents = f.read()

                # ... (with binding_abspath)

                self._compat2binding[binding_compat, _binding_bus(binding)] = \
                    (binding, binding_relpath)

zip() is for iterating both lists at the same time.

binding_paths() would be similar to the old version:

def _binding_paths(bindings_dir):
    # Returns a list of paths to all .yaml files in one 'bindings_dir'

    binding_paths = []

    for subdir, _, filenames in os.walk(bindings_dir):
        for filename in filenames:
            if filename.endswith(".yaml"):
                binding_paths.append(os.path.join(subdir, filename))

    return binding_paths

Thoughts?

It seems to indirectly remove the ./ at the start of paths too, so would need a test suite update.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why this is better than a dict but why not, LGTM.

One of things I liked about the dict is that... it could be done in a small number of changed lines which is easier to rebase constantly. Not the best of reasons, granted :-)


self._compat2binding = {}
for binding_path in self._binding_paths:
Expand Down Expand Up @@ -212,7 +221,7 @@ def _init_compat2binding(self, bindings_dirs):
_check_binding(binding, binding_path)

self._compat2binding[binding_compat, _binding_bus(binding)] = \
(binding, binding_path)
(binding, relpaths[binding_path])

def _merge_included_bindings(self, binding, binding_path):
# Merges any bindings listed in the 'include:' section of 'binding'
Expand Down Expand Up @@ -1292,17 +1301,17 @@ def _dt_compats(dt):
for compat in node.props["compatible"].to_strings()}


def _binding_paths(bindings_dirs):
# Returns a list with the paths to all bindings (.yaml files) in
# 'bindings_dirs'
def _binding_rel_yamls(bindings_dir):
# Returns a list of all the relative paths to all .yaml files in one
# 'bindings_dir'

binding_paths = []

for bindings_dir in bindings_dirs:
for root, _, filenames in os.walk(bindings_dir):
for filename in filenames:
if filename.endswith(".yaml"):
binding_paths.append(os.path.join(root, filename))
for subdir, _, filenames in os.walk(bindings_dir):
for filename in filenames:
if filename.endswith(".yaml"):
reldir = os.path.relpath(subdir, bindings_dir)
binding_paths.append(os.path.join(reldir, filename))

return binding_paths

Expand Down
19 changes: 17 additions & 2 deletions scripts/dts/gen_defines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
# edtlib. This will keep this script simple.

import argparse
import os
from pathlib import Path
import sys

import edtlib
Expand All @@ -40,7 +42,20 @@ def main():

out_comment("Generated by gen_defines.py", blank_before=False)
out_comment("DTS input file: " + args.dts, blank_before=False)
out_comment("Directories with bindings: " + ", ".join(args.bindings_dirs),

zbase = Path(os.environ["ZEPHYR_BASE"])
shorter_bdirs = {}
for d in args.bindings_dirs:
try:
shorter_bdirs[d] = Path(d).relative_to(zbase)
except ValueError:
# leave as is when outside ZEPHYR_BASE
shorter_bdirs[d] = Path(d)

out_comment("Directories with bindings: " +
", ".join([ str("ZEPHYR_BASE" / d)
for d in shorter_bdirs.values()
]),
Copy link
Collaborator

@ulfalizer ulfalizer Sep 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Poked around for a while and came up with a helper function that might be handy here:

    out_comment("Generated by gen_defines.py", blank_before=False)
    out_comment("DTS input file: " + args.dts, blank_before=False)
    out_comment("Directories with bindings: " +
                ", ".join(map(relativize, args.bindings_dirs)),
                blank_before=False)

Definition (I put it after parse_args()):

def relativize(path):
    # If 'path' is within $ZEPHYR_BASE, returns it relative to $ZEPHYR_BASE,
    # with a "$ZEPHYR_BASE/..." hint at the start of the string. Otherwise,
    # returns 'path' unchanged.

    if "ZEPHYR_BASE" not in os.environ:
        return path

    try:
        return str("$ZEPHYR_BASE" /
                   Path(path).relative_to(os.environ["ZEPHYR_BASE"]))
    except ValueError:
        # Not within ZEPHYR_BASE
        return path

Has the advantage that gen_defines.py won't crash if $ZEPHYR_BASE is unset, though I don't know if that'd come up.

Thoughts?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can also do:

zbase = os.environ.get("ZEPHYR_BASE")
if not zbase:
   return path

... / Path(path).relative_to(zbase)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, bit neater.

blank_before=False)

active_compats = set()
Expand All @@ -53,7 +68,7 @@ def main():
continue

out_comment("Device tree node: " + dev.path)
out_comment("Binding (compatible = {}): {}".format(
out_comment("DTS Binding (compatible = {}): {}".format(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remembered that DTS is really the name of the .dts source format, so maybe just "Binding" is fine.

dev.matching_compat, dev.binding_path),
blank_before=False)
out_comment("Binding description: " + dev.description,
Expand Down
32 changes: 16 additions & 16 deletions scripts/dts/testedtlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,47 +41,47 @@ def verify_streq(actual, expected):
#

verify_streq(edt.get_dev("/interrupt-parent-test/node").interrupts,
"[<Interrupt, name: foo, target: <Device /interrupt-parent-test/controller in 'test.dts', binding test-bindings/interrupt-3-cell.yaml>, specifier: {'one': 1, 'two': 2, 'three': 3}>, <Interrupt, name: bar, target: <Device /interrupt-parent-test/controller in 'test.dts', binding test-bindings/interrupt-3-cell.yaml>, specifier: {'one': 4, 'two': 5, 'three': 6}>]")
"[<Interrupt, name: foo, target: <Device /interrupt-parent-test/controller in 'test.dts', binding ./interrupt-3-cell.yaml>, specifier: {'one': 1, 'two': 2, 'three': 3}>, <Interrupt, name: bar, target: <Device /interrupt-parent-test/controller in 'test.dts', binding ./interrupt-3-cell.yaml>, specifier: {'one': 4, 'two': 5, 'three': 6}>]")

verify_streq(edt.get_dev("/interrupts-extended-test/node").interrupts,
"[<Interrupt, target: <Device /interrupts-extended-test/controller-0 in 'test.dts', binding test-bindings/interrupt-1-cell.yaml>, specifier: {'one': 1}>, <Interrupt, target: <Device /interrupts-extended-test/controller-1 in 'test.dts', binding test-bindings/interrupt-2-cell.yaml>, specifier: {'one': 2, 'two': 3}>, <Interrupt, target: <Device /interrupts-extended-test/controller-2 in 'test.dts', binding test-bindings/interrupt-3-cell.yaml>, specifier: {'one': 4, 'two': 5, 'three': 6}>]")
"[<Interrupt, target: <Device /interrupts-extended-test/controller-0 in 'test.dts', binding ./interrupt-1-cell.yaml>, specifier: {'one': 1}>, <Interrupt, target: <Device /interrupts-extended-test/controller-1 in 'test.dts', binding ./interrupt-2-cell.yaml>, specifier: {'one': 2, 'two': 3}>, <Interrupt, target: <Device /interrupts-extended-test/controller-2 in 'test.dts', binding ./interrupt-3-cell.yaml>, specifier: {'one': 4, 'two': 5, 'three': 6}>]")

verify_streq(edt.get_dev("/interrupt-map-test/node@0").interrupts,
"[<Interrupt, target: <Device /interrupt-map-test/controller-0 in 'test.dts', binding test-bindings/interrupt-1-cell.yaml>, specifier: {'one': 0}>, <Interrupt, target: <Device /interrupt-map-test/controller-1 in 'test.dts', binding test-bindings/interrupt-2-cell.yaml>, specifier: {'one': 0, 'two': 1}>, <Interrupt, target: <Device /interrupt-map-test/controller-2 in 'test.dts', binding test-bindings/interrupt-3-cell.yaml>, specifier: {'one': 0, 'two': 0, 'three': 2}>]")
"[<Interrupt, target: <Device /interrupt-map-test/controller-0 in 'test.dts', binding ./interrupt-1-cell.yaml>, specifier: {'one': 0}>, <Interrupt, target: <Device /interrupt-map-test/controller-1 in 'test.dts', binding ./interrupt-2-cell.yaml>, specifier: {'one': 0, 'two': 1}>, <Interrupt, target: <Device /interrupt-map-test/controller-2 in 'test.dts', binding ./interrupt-3-cell.yaml>, specifier: {'one': 0, 'two': 0, 'three': 2}>]")

verify_streq(edt.get_dev("/interrupt-map-test/node@1").interrupts,
"[<Interrupt, target: <Device /interrupt-map-test/controller-0 in 'test.dts', binding test-bindings/interrupt-1-cell.yaml>, specifier: {'one': 3}>, <Interrupt, target: <Device /interrupt-map-test/controller-1 in 'test.dts', binding test-bindings/interrupt-2-cell.yaml>, specifier: {'one': 0, 'two': 4}>, <Interrupt, target: <Device /interrupt-map-test/controller-2 in 'test.dts', binding test-bindings/interrupt-3-cell.yaml>, specifier: {'one': 0, 'two': 0, 'three': 5}>]")
"[<Interrupt, target: <Device /interrupt-map-test/controller-0 in 'test.dts', binding ./interrupt-1-cell.yaml>, specifier: {'one': 3}>, <Interrupt, target: <Device /interrupt-map-test/controller-1 in 'test.dts', binding ./interrupt-2-cell.yaml>, specifier: {'one': 0, 'two': 4}>, <Interrupt, target: <Device /interrupt-map-test/controller-2 in 'test.dts', binding ./interrupt-3-cell.yaml>, specifier: {'one': 0, 'two': 0, 'three': 5}>]")

verify_streq(edt.get_dev("/interrupt-map-bitops-test/node@70000000E").interrupts,
"[<Interrupt, target: <Device /interrupt-map-bitops-test/controller in 'test.dts', binding test-bindings/interrupt-2-cell.yaml>, specifier: {'one': 3, 'two': 2}>]")
"[<Interrupt, target: <Device /interrupt-map-bitops-test/controller in 'test.dts', binding ./interrupt-2-cell.yaml>, specifier: {'one': 3, 'two': 2}>]")

#
# Test GPIOs
#

verify_streq(edt.get_dev("/gpio-test/node").gpios,
"{'': [<GPIO, name: , target: <Device /gpio-test/controller-0 in 'test.dts', binding test-bindings/gpio-1-cell.yaml>, specifier: {'one': 1}>, <GPIO, name: , target: <Device /gpio-test/controller-1 in 'test.dts', binding test-bindings/gpio-2-cell.yaml>, specifier: {'one': 2, 'two': 3}>], 'foo': [<GPIO, name: foo, target: <Device /gpio-test/controller-1 in 'test.dts', binding test-bindings/gpio-2-cell.yaml>, specifier: {'one': 4, 'two': 5}>], 'bar': [<GPIO, name: bar, target: <Device /gpio-test/controller-1 in 'test.dts', binding test-bindings/gpio-2-cell.yaml>, specifier: {'one': 6, 'two': 7}>]}")
"{'': [<GPIO, name: , target: <Device /gpio-test/controller-0 in 'test.dts', binding ./gpio-1-cell.yaml>, specifier: {'one': 1}>, <GPIO, name: , target: <Device /gpio-test/controller-1 in 'test.dts', binding ./gpio-2-cell.yaml>, specifier: {'one': 2, 'two': 3}>], 'foo': [<GPIO, name: foo, target: <Device /gpio-test/controller-1 in 'test.dts', binding ./gpio-2-cell.yaml>, specifier: {'one': 4, 'two': 5}>], 'bar': [<GPIO, name: bar, target: <Device /gpio-test/controller-1 in 'test.dts', binding ./gpio-2-cell.yaml>, specifier: {'one': 6, 'two': 7}>]}")

#
# Test clocks
#

verify_streq(edt.get_dev("/clock-test/node").clocks,
"[<Clock, name: fixed, frequency: 123, target: <Device /clock-test/fixed-clock in 'test.dts', binding test-bindings/fixed-clock.yaml>, specifier: {}>, <Clock, name: one-cell, target: <Device /clock-test/clock-1 in 'test.dts', binding test-bindings/clock-1-cell.yaml>, specifier: {'one': 1}>, <Clock, name: two-cell, target: <Device /clock-test/clock-2 in 'test.dts', binding test-bindings/clock-2-cell.yaml>, specifier: {'one': 1, 'two': 2}>]")
"[<Clock, name: fixed, frequency: 123, target: <Device /clock-test/fixed-clock in 'test.dts', binding ./fixed-clock.yaml>, specifier: {}>, <Clock, name: one-cell, target: <Device /clock-test/clock-1 in 'test.dts', binding ./clock-1-cell.yaml>, specifier: {'one': 1}>, <Clock, name: two-cell, target: <Device /clock-test/clock-2 in 'test.dts', binding ./clock-2-cell.yaml>, specifier: {'one': 1, 'two': 2}>]")

#
# Test PWMs
#

verify_streq(edt.get_dev("/pwm-test/node").pwms,
"[<PWM, name: zero-cell, target: <Device /pwm-test/pwm-0 in 'test.dts', binding test-bindings/pwm-0-cell.yaml>, specifier: {}>, <PWM, name: one-cell, target: <Device /pwm-test/pwm-1 in 'test.dts', binding test-bindings/pwm-1-cell.yaml>, specifier: {'one': 1}>]")
"[<PWM, name: zero-cell, target: <Device /pwm-test/pwm-0 in 'test.dts', binding ./pwm-0-cell.yaml>, specifier: {}>, <PWM, name: one-cell, target: <Device /pwm-test/pwm-1 in 'test.dts', binding ./pwm-1-cell.yaml>, specifier: {'one': 1}>]")

#
# Test IO channels
#

verify_streq(edt.get_dev("/io-channel-test/node").iochannels,
"[<IOChannel, name: io-channel, target: <Device /io-channel-test/io-channel in 'test.dts', binding test-bindings/io-channel.yaml>, specifier: {'one': 1}>]")
"[<IOChannel, name: io-channel, target: <Device /io-channel-test/io-channel in 'test.dts', binding ./io-channel.yaml>, specifier: {'one': 1}>]")

#
# Test 'reg'
Expand Down Expand Up @@ -131,10 +131,10 @@ def verify_streq(actual, expected):
#

verify_streq(edt.get_dev("/buses/foo-bus/node").binding_path,
"test-bindings/device-on-foo-bus.yaml")
"./device-on-foo-bus.yaml")

verify_streq(edt.get_dev("/buses/bar-bus/node").binding_path,
"test-bindings/device-on-bar-bus.yaml")
"./device-on-bar-bus.yaml")

#
# Test 'child-binding:'
Expand All @@ -144,15 +144,15 @@ def verify_streq(actual, expected):
child2 = edt.get_dev("/child-binding/child-2")
grandchild = edt.get_dev("/child-binding/child-1/grandchild")

verify_streq(child1.binding_path, "test-bindings/child-binding.yaml")
verify_streq(child1.binding_path, "./child-binding.yaml")
verify_streq(child1.description, "child node")
verify_streq(child1.props, "{'child-prop': <Property, name: child-prop, type: int, value: 1>}")

verify_streq(child2.binding_path, "test-bindings/child-binding.yaml")
verify_streq(child2.binding_path, "./child-binding.yaml")
verify_streq(child2.description, "child node")
verify_streq(child2.props, "{'child-prop': <Property, name: child-prop, type: int, value: 3>}")

verify_streq(grandchild.binding_path, "test-bindings/child-binding.yaml")
verify_streq(grandchild.binding_path, "./child-binding.yaml")
verify_streq(grandchild.description, "grandchild node")
verify_streq(grandchild.props, "{'grandchild-prop': <Property, name: grandchild-prop, type: int, value: 2>}")

Expand Down Expand Up @@ -184,10 +184,10 @@ def verify_streq(actual, expected):
edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"])

verify_streq(edt.get_dev("/in-dir-1").binding_path,
"test-bindings/multidir.yaml")
"./multidir.yaml")

verify_streq(edt.get_dev("/in-dir-2").binding_path,
"test-bindings-2/multidir.yaml")
"./multidir.yaml")


print("all tests passed")
Expand Down