gen_kobject_list.py: OrderedDict for < 3.6 determinism

Dictionaries are iterated in a random order by Python 3.5 and before.
This could have caused "Unstable" CI in PR #13921 and maybe others.
Anyway we want builds to be determimistic by default. Explicit
randomness can be added for better coverage but not by default.

1. When running "make kobj_types_h_target" repeatedly one can observe
that the following .h files keep changing in
build/zephyr/include/generated/:

- kobj-types-enum.h
- otype-to-str.h
- otype-to-size.h

Switching kobjects to OrderedDict makes these 3 .h files deterministic.

2. When running this test repeatedly with CONFIG_USERSPACE=y:

  rm build/zephyr/*.gperf && make -C build obj_list

... the dict used for --gperf-output seems to be deterministic, probably
because its keys are all integers (memory addresses). However we can't
take that for granted with Python < 3.6 so out of caution also switch
the output of find_objects() in elf_helper.py to a sorted OrderedDict.

PS: I would normally prefer official Python documentation to
StackOverflow however this one is a good summary and has all the
multiple pointers to the... official Python documentation.

Signed-off-by: Marc Herbert <[email protected]>

scripts/elf_helper.py

@@ -9,6 +9,8 @@
 import struct
 from distutils.version import LooseVersion
 
+from collections import OrderedDict
+
 import elftools
 from elftools.elf.elffile import ELFFile
 from elftools.elf.sections import SymbolTableSection
@@ -518,7 +520,14 @@ def find_kobjects(self, syms):
             ret[addr] = ko
 
         self.debug("found %d kernel object instances total" % len(ret))
-        return ret
+
+        # 1. Before python 3.7 dict order is not guaranteed. With Python
+        #    3.5 it doesn't seem random with *integer* keys but can't
+        #    rely on that.
+        # 2. OrderedDict means _insertion_ order, so not enough because
+        #    built from other (random!) dicts: need to _sort_ first.
+        # 3. Sorting memory address looks good.
+        return OrderedDict(sorted(ret.items()))
 
     def get_symbols(self):
         for section in self.elf.iter_sections():

scripts/gen_kobject_list.py

@@ -11,26 +11,33 @@
 import struct
 from elf_helper import ElfHelper, kobject_to_enum
 
+from collections import OrderedDict
+
 # Keys in this dictionary are structs which should be recognized as kernel
 # objects. Values should either be None, or the name of a Kconfig that
 # indicates the presence of this object's definition in case it is not
 # available in all configurations.
 
-kobjects = {
-    "k_mem_slab": None,
-    "k_msgq": None,
-    "k_mutex": None,
-    "k_pipe": None,
-    "k_queue": None,
-    "k_poll_signal": None,
-    "k_sem": None,
-    "k_stack": None,
-    "k_thread": None,
-    "k_timer": None,
-    "_k_thread_stack_element": None,
-    "net_context": "CONFIG_NETWORKING",
-    "device": None
-}
+# Regular dictionaries are ordered only with Python 3.6 and
+# above. Good summary and pointers to official documents at:
+# https://stackoverflow.com/questions/39980323/are-dictionaries-ordered-in-python-3-6
+kobjects = OrderedDict ([
+    ("k_mem_slab", None),
+    ("k_msgq", None),
+    ("k_mutex", None),
+    ("k_pipe", None),
+    ("k_queue", None),
+    ("k_poll_signal", None),
+    ("k_sem", None),
+    ("k_stack", None),
+    ("k_thread", None),
+    ("k_timer", None),
+    ("_k_thread_stack_element", None),
+    ("net_context", "CONFIG_NETWORKING"),
+    ("device", None),
+])
+
+
 
 subsystems = [
     "adc_driver_api",
@@ -269,10 +276,14 @@ def main():
     parse_args()
 
     if args.gperf_output:
+        assert args.kernel, "--kernel ELF required for --gperf-output"
         eh = ElfHelper(args.kernel, args.verbose, kobjects, subsystems)
         syms = eh.get_symbols()
         max_threads = syms["CONFIG_MAX_THREAD_BYTES"] * 8
         objs = eh.find_kobjects(syms)
+        if not objs:
+            sys.stderr.write("WARNING: zero kobject found in %s\n"
+                             % args.kernel)
 
         thread_counter = eh.get_thread_counter()
         if thread_counter > max_threads:

scripts/gen_priv_stacks.py

@@ -105,6 +105,9 @@ def main():
     syms = eh.get_symbols()
     max_threads = syms["CONFIG_MAX_THREAD_BYTES"] * 8
     objs = eh.find_kobjects(syms)
+    if not objs:
+        sys.stderr.write("WARNING: zero kobject found in %s\n"
+                         % args.kernel)
 
     thread_counter = eh.get_thread_counter()
     if thread_counter > max_threads: