Merge pull request #3677 from randombit/jack/ninja-build

.github/workflows/ci.yml

@@ -56,7 +56,7 @@ jobs:
           arch: ${{ matrix.arch }}
 
       - name: Build and Test Botan
-        run: python3 ./src/scripts/ci_build.py --cc='msvc' --make-tool='jom' --cpu='${{ matrix.arch }}' --test-results-dir=junit_results ${{ matrix.target }}
+        run: python3 ./src/scripts/ci_build.py --cc='msvc' --make-tool='ninja' --cpu='${{ matrix.arch }}' --test-results-dir=junit_results ${{ matrix.target }}
 
   linux:
     name: "Linux"
@@ -147,7 +147,7 @@ jobs:
           - target: sanitizer
             compiler: msvc
             host_os: windows-2022
-            make_tool: jom
+            make_tool: ninja
           - target: sanitizer
             compiler: clang
             host_os: ubuntu-22.04

.gitignore

@@ -1,5 +1,7 @@
 /Makefile
 CMakeLists.txt*
+build.ninja
+.ninja_log
 libbotan*.so.*
 *.a
 *.so

configure.py

@@ -519,6 +519,9 @@ def process_command_line(args):
     build_group.add_option('--build-targets', default=None, dest="build_targets", action='append',
                            help="build specific targets and tools (%s)" % ', '.join(ACCEPTABLE_BUILD_TARGETS))
 
+    build_group.add_option('--build-tool', default='make',
+                           help="specify the build tool (make, ninja)")
+
     build_group.add_option('--with-pkg-config', action='store_true', default=None,
                            help=optparse.SUPPRESS_HELP)
     build_group.add_option('--without-pkg-config', dest='with_pkg_config', action='store_false',
@@ -3152,6 +3155,9 @@ def validate_options(options, info_os, info_cc, available_module_policies):
         if options.build_fuzzers == 'klee' and options.os != 'llvm':
             raise UserError('Building for KLEE requires targeting LLVM')
 
+    if options.build_tool not in ['make', 'ninja']:
+        raise UserError("Unknown --build-tool option (possible values: make, ninja)")
+
     if options.build_static_lib is False and options.build_shared_lib is False:
         raise UserError('With both --disable-static-library and --disable-shared-library, nothing to do')
 
@@ -3290,12 +3296,16 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou
             if ex.errno != errno.EEXIST:
                 logging.error('Error while creating "%s": %s', build_dir, ex)
 
-    def write_template_with_variables(sink, template, variables):
+    def write_template_with_variables(sink, template, variables, postproc_fn = None):
+        output = process_template(template, variables)
+        if postproc_fn:
+            output = postproc_fn(output)
+
         with open(sink, 'w', encoding='utf8') as f:
-            f.write(process_template(template, variables))
+            f.write(output)
 
-    def write_template(sink, template):
-        write_template_with_variables(sink, template, template_vars)
+    def write_template(sink, template, postproc_fn = None):
+        write_template_with_variables(sink, template, template_vars, postproc_fn)
 
     def in_build_dir(p):
         return os.path.join(build_paths.build_dir, p)
@@ -3351,7 +3361,25 @@ def link_headers(headers, visibility, directory):
     if options.with_compilation_database:
         write_template(in_build_dir('compile_commands.json'), in_build_data('compile_commands.json.in'))
 
-    write_template(template_vars['makefile_path'], in_build_data('makefile.in'))
+    if options.build_tool == 'make':
+        write_template(template_vars['makefile_path'], in_build_data('makefile.in'))
+    elif options.build_tool == 'ninja':
+        def escape_build_lines(contents):
+            ninja_build_line = re.compile('^build (.*): (.*)')
+
+            output = []
+            for line in contents.split('\n'):
+                match = ninja_build_line.match(line)
+                if match:
+                    escaped1 = match.group(1).replace(':', '$:')
+                    escaped2 = match.group(2).replace(':', '$:')
+                    output.append('build %s: %s' % (escaped1, escaped2))
+                else:
+                    output.append(line)
+
+            return "\n".join(output)
+
+        write_template(template_vars['ninja_build_path'], in_build_data('ninja.in'), escape_build_lines)
 
     if options.with_doxygen:
         for module_name, info in info_modules.items():

doc/building.rst

@@ -181,8 +181,8 @@ On macOS
 
 A build on macOS works much like that on any other Unix-like system.
 
-To build a universal binary for macOS, for older macOs releases, 
-you need to set some additional build flags. 
+To build a universal binary for macOS, for older macOs releases,
+you need to set some additional build flags.
 Do this with the `configure.py` flag `--cc-abi-flags`::
 
   --cc-abi-flags="-force_cpusubtype_ALL -mmacosx-version-min=10.4 -arch i386 -arch ppc"
@@ -209,8 +209,17 @@ shell), and run::
    $ nmake check
    $ nmake install
 
-Botan supports the nmake replacement `Jom <https://wiki.qt.io/Jom>`_
-which enables you to run multiple build jobs in parallel.
+Micosoft's ``nmake`` does not support building multiple jobs in parallel, which
+is unfortunate when building on modern multicore machines. It is possible to use
+the (somewhat unmaintained) `Jom <https://wiki.qt.io/Jom>`_ build tool, which is
+a ``nmake`` compatible build system that supports parallel builds. Alternately,
+starting in Botan 3.2, there is additionally support for using the ``ninja``
+build tool as an alternative to ``nmake``::
+
+   $ python3 configure.py --cc=msvc --os=windows --build-tool=ninja
+   $ ninja
+   $ ninja check
+   $ ninja install
 
 For MinGW, use::
 
@@ -226,6 +235,17 @@ compiler to look for both include files and library files in
 place where they will be in the default compiler search paths (consult
 your documentation and/or local expert for details).
 
+Ninja Support
+---------------
+
+Starting in Botan 3.2, there is additionally support for the
+`ninja <https://ninja-build.org>`_ build system.
+
+This is particularly useful on Windows as there the default build tool ``nmake``
+does not support parallel jobs. The ``ninja`` based build also works on Unix and
+macOs systems.
+
+Support for ``ninja`` is still new and there are probably some rough edges.
 
 For iOS using XCode
 -------------------------

src/build-data/ninja.in

@@ -0,0 +1,184 @@
+ninja_required_version = 1.3
+
+CXX            = %{cxx}
+LINKER         = %{linker}
+PYTHON_EXE     = %{python_exe}
+
+ABI_FLAGS      = %{cc_sysroot} %{cxx_abi_flags}
+LANG_FLAGS     = %{cc_lang_flags} %{os_feature_macros}
+LANG_EXE_FLAGS = %{cc_lang_binary_linker_flags}
+CXXFLAGS       = %{cc_compile_flags} -DBOTAN_IS_BEING_BUILT
+WARN_FLAGS     = %{cc_warning_flags}
+
+LDFLAGS        = %{ldflags}
+
+EXE_LINK_CMD   = %{exe_link_cmd}
+
+LIB_LINKS_TO        = %{external_link_cmd} %{link_to}
+BUILD_DIR_LINK_PATH = %{build_dir_link_path}
+EXE_LINKS_TO        = %{link_to_botan} ${LIB_LINKS_TO} %{extra_libs}
+
+SCRIPTS_DIR    = %{scripts_dir}
+INSTALLED_LIB_DIR = %{libdir}
+
+rule compile_lib
+  command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out
+
+rule compile_exe
+  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out
+
+# The primary target
+build all: phony %{all_targets}
+default all
+
+# Library targets
+
+%{if build_static_lib}
+
+rule link_static
+  command = %{ar_command} %{ar_options} %{ar_output_to}$out $in
+
+build %{out_dir}/%{static_lib_name}: link_static %{join lib_objs}
+
+%{endif}
+%{if build_shared_lib}
+
+rule link_shared
+  command = %{lib_link_cmd} ${ABI_FLAGS} ${LDFLAGS} $in ${LIB_LINKS_TO} %{output_to_exe}$out
+
+build %{out_dir}/%{shared_lib_name}: link_shared %{join lib_objs}
+%{endif}
+%{if symlink_shared_lib}
+
+rule symlink
+  command = cd %{out_dir} && ln -fs $in $out
+
+build %{out_dir}/%{soname_base}: symlink %{out_dir}/%{shared_lib_name}
+build %{out_dir}/%{soname_patch}: symlink %{out_dir}/%{shared_lib_name}
+
+%{endif}
+
+rule link_cli
+  command = ${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} ${EXE_LINKS_TO} %{output_to_exe}$out
+
+rule link_tests
+  command = ${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} %{test_exe_extra_ldflags} ${EXE_LINKS_TO} %{output_to_exe}$out
+
+
+# Executable targets
+
+build %{cli_exe}: link_cli %{join cli_objs} | %{library_targets}
+
+build %{test_exe}: link_tests %{join test_objs} | %{library_targets}
+
+%{if build_fuzzers}
+
+build fuzzers: phony %{fuzzer_bin}
+
+rule fuzzer_corpus
+  command = git clone --depth=1 https://github.com/randombit/crypto-corpus.git fuzzer_corpus
+
+build fuzzer_corpus: fuzzer_corpus
+
+rule fuzzer_corpus_zip
+  command = %{base_dir}/src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir}
+
+build fuzzer_corpus_zip: fuzzer_corpus_zip | fuzzer_corpus
+
+rule link_fuzzer
+  command =	${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} ${EXE_LINKS_TO} %{fuzzer_lib} %{output_to_exe}$out
+
+%{endif}
+
+%{if build_examples}
+
+build examples: link_cli %{example_bin} | %{library_targets}
+
+%{endif}
+
+
+%{if build_bogo_shim}
+
+build bogo_shim: link_cli botan_bogo_shim | %{library_targets}
+
+# BoGo shim
+build %{out_dir}/botan_bogo_shim: compile_exe %{bogo_shim_src}
+
+%{endif}
+
+# Misc targets
+
+rule build_docs
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/build_docs.py" --build-dir="%{build_dir}"
+
+build %{doc_stamp_file}: build_docs
+
+rule clean
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/cleanup.py" --build-dir="%{build_dir}"
+
+rule distclean
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/cleanup.py" --build-dir="%{build_dir}" --distclean
+
+rule install
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/install.py" --build-dir="%{build_dir}"
+
+rule check
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/check.py" --build-dir="%{build_dir}"
+
+rule fmt
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/dev_tools/run_clang_format.py"
+
+rule tidy
+  command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/dev_tools/run_clang_tidy.py" --only-changed-files
+
+# Target aliases
+
+build cli: phony %{cli_exe}
+
+build tests: phony %{test_exe}
+
+build libs: phony %{library_targets} $
+%{if symlink_shared_lib}
+  %{out_dir}/%{soname_base} %{out_dir}/%{soname_patch}
+%{endif}
+
+build docs: phony %{doc_stamp_file}
+
+build clean: clean
+
+build distclean: distclean
+
+build install: install | %{install_targets}
+
+build check: check | tests
+
+build fmt: fmt
+
+build tidy: tidy
+
+
+# Build Commands
+
+%{for lib_build_info}
+build %{obj}: compile_lib %{src}
+%{endfor}
+
+%{for cli_build_info}
+build %{obj}: compile_exe %{src}
+%{endfor}
+
+%{for test_build_info}
+build %{obj}: compile_exe %{src}
+%{endfor}
+
+%{for fuzzer_build_info}
+build %{obj}: compile_exe %{src}
+
+build %{exe}: link_fuzzer %{obj} | %{library_targets}
+%{endfor}
+
+%{for examples_build_info}
+build %{obj}: compile_exe %{src}
+
+build %{exe}: link_cli %{obj} | %{library_targets}
+%{endfor}

src/editors/vscode/tasks.json

@@ -42,7 +42,7 @@
                 "problemMatcher": "$gcc"
             },
             "windows": {
-                "command": "jom",
+                "command": "ninja",
                 "problemMatcher": "$msCompile"
             },
             "presentation": {

src/scripts/ci/setup_gh_actions.ps1

@@ -2,6 +2,7 @@
 #
 # (C) 2022 Jack Lloyd
 # (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
+# (C) 2023 René Fischer, Rohde & Schwarz Cybersecurity
 #
 # Botan is released under the Simplified BSD License (see license.txt)
 
@@ -11,7 +12,6 @@ param(
     [String]$ARCH
 )
 
-choco install -y jom
 choco install -y sccache
 
 # find the sccache cache location and store it in the build job's environment

src/scripts/ci_build.py

@@ -530,18 +530,14 @@ def validate_make_tool(make_tool, build_jobs):
     if make_tool == '':
         return validate_make_tool('make', build_jobs)
 
-    if make_tool not in ['nmake', 'jom', 'make']:
+    if make_tool not in ['ninja', 'nmake', 'make']:
         raise Exception("Don't know about %s as a make tool" % (make_tool))
 
-    # Hack to work around jom occasionally failing to install
-    # https://github.com/randombit/botan/issues/3629
-    if make_tool == 'jom' and not have_prog('jom'):
-        return ['nmake']
+    if make_tool in ['make']:
+        return make_tool, ['-j%d' % (build_jobs), '-k']
 
-    if make_tool in ['make', 'jom']:
-        return [make_tool, '-j%d' % (build_jobs)]
     else:
-        return [make_tool]
+        return make_tool, []
 
 def main(args=None):
     """
@@ -654,13 +650,17 @@ def main(args=None):
             options.pkcs11_lib, options.use_gdb, options.disable_werror,
             options.extra_cxxflags, options.disabled_tests)
 
-        cmds.append([py_interp, os.path.join(root_dir, 'configure.py')] + config_flags)
+        make_tool, make_opts = validate_make_tool(options.make_tool, options.build_jobs)
 
-        make_cmd = validate_make_tool(options.make_tool, options.build_jobs)
-        if build_dir != '.':
-            make_cmd = ['indir:%s' % build_dir] + make_cmd
+        cmds.append([py_interp,
+            os.path.join(root_dir, 'configure.py')] +
+            ['--build-tool=' + make_tool] +
+            config_flags)
 
-        make_cmd += ['-k']
+        make_cmd = [make_tool] + make_opts
+
+        if build_dir != '.':
+            make_cmd = ['indir:%s' % build_dir] + [make_tool] + make_opts
 
         if target == 'docs':
             cmds.append(make_cmd + ['docs'])