script: test framework to validate the reproducible builds

A test framework to verify that builds are reproducible, i.e.
producing exactly the same binary. Validating the binary generated
from same directory, different directories and on different hosts
with same tool chain.

Signed-off-by: Ajay Kishore <[email protected]>

scripts/check_reproducible_builds.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This script verify that builds are reproducible, i.e.
+# producing exactly the same binary if built using the same code base.
+# Using diffoscope tool which recursively transform various binary
+# formats into more human-readable forms for comparison.
+#
+# Command to use test script:
+# python check_reproducible_builds.py -i1 path_of_elf1 -i2 path_of_elf2
+
+import os
+import sys
+import time
+import re
+import subprocess
+import argparse
+import colorama
+
+def reproducible_builds(input_file1, input_file2):
+    res = res1 = res2 = res3 = 0
+    filename  = os.path.join('/tmp', 'out.txt')
+
+    if os.path.exists("filename"):
+        os.remove("filename")
+
+    if os.path.exists(input_file1):
+        if os.path.exists(input_file2):
+            subprocess.call("diffoscope --text filename {0} {1}". format(input_file1, input_file2), shell=True)
+
+    if os.path.isfile('filename'):
+        with open('filename', 'r') as out_file:
+            for line in out_file:
+                if 'Start of section headers:' in line:
+                    if res is 0:
+                        res = int(re.search(r'\d+', line).group())
+                    else:
+                        res1 = int(re.search(r'\d+', line).group())
+                        print(colorama.Fore.LIGHTRED_EX + 'Bytes in section header mismatch')
+                        print(colorama.Fore.LIGHTRED_EX + 'Bytes in section header of elf file1 and file2 are {0} and  {1} respectively'.format(res, res1) + colorama.Fore.RESET)
+                        res = res1 = 0
+                        break
+            out_file.seek(0, 0);
+            for line in out_file:
+                if 'Number of section headers:' in line:
+                    if res is 0:
+                        res = int(re.search(r'\d+', line).group())
+                    else:
+                        res1 = int(re.search(r'\d+', line).group())
+                        print(colorama.Fore.LIGHTRED_EX + 'Number of section headers mismatch')
+                        print(colorama.Fore.LIGHTRED_EX + 'Number of section headers of elf file1 and file2 are {0} and  {1} respectively'.format(res, res1)  + colorama.Fore.RESET)
+                        res = res1 = 0
+                        break
+
+            out_file.seek(0, 0);
+            for line in out_file:
+                if 'Symbol table \'.symtab\' contains' in line:
+                    if res is 0:
+                        res = int(re.search(r'\d+', line).group())
+                    else:
+                        res1 = int(re.search(r'\d+', line).group())
+                        print(colorama.Fore.LIGHTRED_EX + 'Entries in symbol table mismatch')
+                        print(colorama.Fore.LIGHTRED_EX + 'Entries in Symbol table of elf file1 and file2 are {0} and  {1} respectively'.format(res, res1) + colorama.Fore.RESET)
+                        break
+    else:
+        print "No difference found in the comparison of both input elf file"
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument(
+        "-i1",
+        "--input1",
+        required=True,
+        help="First ELF file to compare")
+    parser.add_argument(
+        "-i2",
+        "--input2",
+        required=True,
+        help="Second Input file to compare")
+
+    args = parser.parse_args()
+
+    ret = reproducible_builds(args.input1, args.input2)
+    sys.exit(ret)

scripts/requirements.txt

@@ -13,6 +13,7 @@ pyelftools==0.24
 pyocd==0.12.0
 pyserial
 pykwalify
+diffoscope
 # "win32" is used for 64-bit Windows as well
 windows-curses; sys_platform == "win32"
 colorama

tests/reproducible_builds/README.txt

@@ -0,0 +1,29 @@
+Title: Test framework to validate reproducible binary
+
+Description:
+This script verify that builds are reproducible, i.e. producing exactly
+the same binary if built using the same source code repository.
+Validating the binary generated in below scenario:
+- build generated in the same directory
+- build generated in different directories
+- Enable “TIMER_RANDOM_GENERATOR" config and generate the build
+
+Usage:
+Execute sh ./check_reproducible_build.sh  which will clone the latest Zephyr
+source code repository and generate the binary in above mentioned scenario.
+And further it will compare all the binaries using diffoscope tool.
+
+Sample Output:
+checking reproducible build from the same directory
+
+checking reproducible build from the different directory
+Bytes in section header mismatch
+Bytes in section header of elf file1 and file2 are 585232 and  548764 respectively
+Number of section headers mismatch
+Number of section headers of elf file1 and file2 are 28 and  27 respectively
+Entries in symbol table mismatch
+Entries in Symbol table of elf file1 and file2 are 981 and  940 respectively
+
+checking reproducible build with CONFIG_TIMER_RANDOM_GENERATOR
+Bytes in section header mismatch
+Bytes in section header of elf file1 and file2 are 548760 and  548764 respectively

tests/reproducible_builds/check_reproducible_build.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+if [ -d "/tmp/zephyr_src1" ]; then
+    rm -rf /tmp/zephyr_src1/
+fi
+
+mkdir /tmp/zephyr_src1
+echo "Cloning latest code in zephyr_src1"
+cd /tmp/zephyr_src1/
+git clone https://github.com/zephyrproject-rtos/zephyr.git
+
+cd /tmp/zephyr_src1/zephyr/
+source /tmp/zephyr_src1/zephyr/zephyr-env.sh
+
+cd samples/hello_world/
+if [ -d "build" ]; then
+    rm -rf build
+fi
+
+mkdir build && cd build
+cmake -GNinja -DBOARD=qemu_x86 ../
+ninja
+
+if [ ! -d "/tmp/zephyr_src1/temp" ]
+then
+    mkdir /tmp/zephyr_src1/temp
+fi
+
+# copy zephyr.elf into a temp folder
+cp /tmp/zephyr_src1/zephyr/samples/hello_world/build/zephyr/zephyr.elf \
+   /tmp/zephyr_src1/temp/
+
+
+cd /tmp/zephyr_src1/zephyr/samples/hello_world/
+if [ -d "build" ]; then
+    rm -rf build
+fi
+
+mkdir build && cd build
+cmake -GNinja -DBOARD=qemu_x86 ../
+ninja
+
+if [ ! -d "/tmp/zephyr_src1/temp1" ]
+then
+    mkdir /tmp/zephyr_src1/temp1/
+fi
+
+# copy zephyr.elf into a temp folder
+cp /tmp/zephyr_src1/zephyr/samples/hello_world/build/zephyr/zephyr.elf \
+   /tmp/zephyr_src1/temp1/
+
+# adding CONFIG_TIMER_RANDOM_GENERATOR into prj.conf
+cd /tmp/zephyr_src1/zephyr/samples/hello_world/
+echo 'CONFIG_TIMER_RANDOM_GENERATOR=y' > temp_prj.conf
+cat prj.conf >> temp_prj.conf
+mv temp_prj.conf prj.conf
+
+if [ -d "build" ]; then
+    rm -rf build
+fi
+mkdir build && cd build
+cmake -GNinja -DBOARD=qemu_x86 ..
+ninja
+
+cd /tmp/zephyr_src1/zephyr/tests/kernel/init/
+if [ -d "build" ]; then
+    rm -rf build
+fi
+mkdir build && cd build
+cmake -GNinja -DBOARD=qemu_x86 ..
+ninja
+
+if [ -f "/tmp/out.txt" ]; then
+    rm -rf out.txt
+fi
+
+echo "checking reproducible build from the same directory"
+python /tmp/zephyr_src1/zephyr/scripts/check_reproducible_builds.py -i1 \
+    /tmp/zephyr_src1/temp/zephyr.elf -i2 /tmp/zephyr_src1/temp1/zephyr.elf
+
+if [ -f "/tmp/out.txt" ]; then
+    rm -rf out.txt
+fi
+
+echo "checking reproducible build from the different directory"
+python /tmp/zephyr_src1/zephyr/scripts/check_reproducible_builds.py -i1 \
+    /tmp/zephyr_src1/zephyr/tests/kernel/init/build/zephyr/zephyr.elf -i2 \
+    /tmp/zephyr_src1/temp1/zephyr.elf
+
+if [ -f "/tmp/out.txt" ]; then
+    rm -rf out.txt
+fi
+
+echo "checking reproducible build with CONFIG_TIMER_RANDOM_GENERATOR"
+python /tmp/zephyr_src1/zephyr/scripts/check_reproducible_builds.py -i1 \
+    /tmp/zephyr_src1/zephyr/samples/hello_world/build/zephyr/zephyr.elf -i2 \
+    /tmp/zephyr_src1/temp/zephyr.elf
+
+#Delete latest cloned code
+rm -rf /tmp/zephyr_src1/
+#Delete the output file generated using diffoscope
+if [ -f "/tmp/out.txt" ]; then
+    rm -rf out.txt
+fi