From a55fc8f4ef4385b884e0244ce425ac4b2c3de6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Fischer?= Date: Sat, 15 Apr 2023 09:01:01 -0400 Subject: [PATCH] Add support for ninja build system --- .github/workflows/ci.yml | 4 +- .gitignore | 2 + configure.py | 38 +++++- doc/building.rst | 28 ++++- src/build-data/ninja.in | 184 ++++++++++++++++++++++++++++ src/editors/vscode/tasks.json | 2 +- src/scripts/ci/setup_gh_actions.ps1 | 2 +- src/scripts/ci_build.py | 26 ++-- 8 files changed, 260 insertions(+), 26 deletions(-) create mode 100644 src/build-data/ninja.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe657ad0fd..aaef4dd993d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: arch: ${{ matrix.arch }} - name: Build and Test Botan - run: python3 ./src/scripts/ci_build.py --cc='msvc' --make-tool='jom' --cpu='${{ matrix.arch }}' --test-results-dir=junit_results ${{ matrix.target }} + run: python3 ./src/scripts/ci_build.py --cc='msvc' --make-tool='ninja' --cpu='${{ matrix.arch }}' --test-results-dir=junit_results ${{ matrix.target }} linux: name: "Linux" @@ -147,7 +147,7 @@ jobs: - target: sanitizer compiler: msvc host_os: windows-2022 - make_tool: jom + make_tool: ninja - target: sanitizer compiler: clang host_os: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index 20297993ab7..97386c4fb6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ /Makefile CMakeLists.txt* +build.ninja +.ninja_log libbotan*.so.* *.a *.so diff --git a/configure.py b/configure.py index 2c7bc9ab345..c6d5d23c824 100755 --- a/configure.py +++ b/configure.py @@ -519,6 +519,9 @@ def process_command_line(args): build_group.add_option('--build-targets', default=None, dest="build_targets", action='append', help="build specific targets and tools (%s)" % ', '.join(ACCEPTABLE_BUILD_TARGETS)) + build_group.add_option('--build-tool', default='make', + help="specify the build tool (make, ninja)") + build_group.add_option('--with-pkg-config', action='store_true', default=None, help=optparse.SUPPRESS_HELP) build_group.add_option('--without-pkg-config', dest='with_pkg_config', action='store_false', @@ -3152,6 +3155,9 @@ def validate_options(options, info_os, info_cc, available_module_policies): if options.build_fuzzers == 'klee' and options.os != 'llvm': raise UserError('Building for KLEE requires targeting LLVM') + if options.build_tool not in ['make', 'ninja']: + raise UserError("Unknown --build-tool option (possible values: make, ninja)") + if options.build_static_lib is False and options.build_shared_lib is False: raise UserError('With both --disable-static-library and --disable-shared-library, nothing to do') @@ -3290,12 +3296,16 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou if ex.errno != errno.EEXIST: logging.error('Error while creating "%s": %s', build_dir, ex) - def write_template_with_variables(sink, template, variables): + def write_template_with_variables(sink, template, variables, postproc_fn = None): + output = process_template(template, variables) + if postproc_fn: + output = postproc_fn(output) + with open(sink, 'w', encoding='utf8') as f: - f.write(process_template(template, variables)) + f.write(output) - def write_template(sink, template): - write_template_with_variables(sink, template, template_vars) + def write_template(sink, template, postproc_fn = None): + write_template_with_variables(sink, template, template_vars, postproc_fn) def in_build_dir(p): return os.path.join(build_paths.build_dir, p) @@ -3351,7 +3361,25 @@ def link_headers(headers, visibility, directory): if options.with_compilation_database: write_template(in_build_dir('compile_commands.json'), in_build_data('compile_commands.json.in')) - write_template(template_vars['makefile_path'], in_build_data('makefile.in')) + if options.build_tool == 'make': + write_template(template_vars['makefile_path'], in_build_data('makefile.in')) + elif options.build_tool == 'ninja': + def escape_build_lines(contents): + ninja_build_line = re.compile('^build (.*): (.*)') + + output = [] + for line in contents.split('\n'): + match = ninja_build_line.match(line) + if match: + escaped1 = match.group(1).replace(':', '$:') + escaped2 = match.group(2).replace(':', '$:') + output.append('build %s: %s' % (escaped1, escaped2)) + else: + output.append(line) + + return "\n".join(output) + + write_template(template_vars['ninja_build_path'], in_build_data('ninja.in'), escape_build_lines) if options.with_doxygen: for module_name, info in info_modules.items(): diff --git a/doc/building.rst b/doc/building.rst index ac95376853b..ef08c1a56ce 100644 --- a/doc/building.rst +++ b/doc/building.rst @@ -181,8 +181,8 @@ On macOS A build on macOS works much like that on any other Unix-like system. -To build a universal binary for macOS, for older macOs releases, -you need to set some additional build flags. +To build a universal binary for macOS, for older macOs releases, +you need to set some additional build flags. Do this with the `configure.py` flag `--cc-abi-flags`:: --cc-abi-flags="-force_cpusubtype_ALL -mmacosx-version-min=10.4 -arch i386 -arch ppc" @@ -209,8 +209,17 @@ shell), and run:: $ nmake check $ nmake install -Botan supports the nmake replacement `Jom `_ -which enables you to run multiple build jobs in parallel. +Micosoft's ``nmake`` does not support building multiple jobs in parallel, which +is unfortunate when building on modern multicore machines. It is possible to use +the (somewhat unmaintained) `Jom `_ build tool, which is +a ``nmake`` compatible build system that supports parallel builds. Alternately, +starting in Botan 3.2, there is additionally support for using the ``ninja`` +build tool as an alternative to ``nmake``:: + + $ python3 configure.py --cc=msvc --os=windows --build-tool=ninja + $ ninja + $ ninja check + $ ninja install For MinGW, use:: @@ -226,6 +235,17 @@ compiler to look for both include files and library files in place where they will be in the default compiler search paths (consult your documentation and/or local expert for details). +Ninja Support +--------------- + +Starting in Botan 3.2, there is additionally support for the +`ninja `_ build system. + +This is particularly useful on Windows as there the default build tool ``nmake`` +does not support parallel jobs. The ``ninja`` based build also works on Unix and +macOs systems. + +Support for ``ninja`` is still new and there are probably some rough edges. For iOS using XCode ------------------------- diff --git a/src/build-data/ninja.in b/src/build-data/ninja.in new file mode 100644 index 00000000000..d6ed2b1f44f --- /dev/null +++ b/src/build-data/ninja.in @@ -0,0 +1,184 @@ +ninja_required_version = 1.3 + +CXX = %{cxx} +LINKER = %{linker} +PYTHON_EXE = %{python_exe} + +ABI_FLAGS = %{cc_sysroot} %{cxx_abi_flags} +LANG_FLAGS = %{cc_lang_flags} %{os_feature_macros} +LANG_EXE_FLAGS = %{cc_lang_binary_linker_flags} +CXXFLAGS = %{cc_compile_flags} -DBOTAN_IS_BEING_BUILT +WARN_FLAGS = %{cc_warning_flags} + +LDFLAGS = %{ldflags} + +EXE_LINK_CMD = %{exe_link_cmd} + +LIB_LINKS_TO = %{external_link_cmd} %{link_to} +BUILD_DIR_LINK_PATH = %{build_dir_link_path} +EXE_LINKS_TO = %{link_to_botan} ${LIB_LINKS_TO} %{extra_libs} + +SCRIPTS_DIR = %{scripts_dir} +INSTALLED_LIB_DIR = %{libdir} + +rule compile_lib + command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out + +rule compile_exe + command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out + +# The primary target +build all: phony %{all_targets} +default all + +# Library targets + +%{if build_static_lib} + +rule link_static + command = %{ar_command} %{ar_options} %{ar_output_to}$out $in + +build %{out_dir}/%{static_lib_name}: link_static %{join lib_objs} + +%{endif} +%{if build_shared_lib} + +rule link_shared + command = %{lib_link_cmd} ${ABI_FLAGS} ${LDFLAGS} $in ${LIB_LINKS_TO} %{output_to_exe}$out + +build %{out_dir}/%{shared_lib_name}: link_shared %{join lib_objs} +%{endif} +%{if symlink_shared_lib} + +rule symlink + command = cd %{out_dir} && ln -fs $in $out + +build %{out_dir}/%{soname_base}: symlink %{out_dir}/%{shared_lib_name} +build %{out_dir}/%{soname_patch}: symlink %{out_dir}/%{shared_lib_name} + +%{endif} + +rule link_cli + command = ${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} ${EXE_LINKS_TO} %{output_to_exe}$out + +rule link_tests + command = ${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} %{test_exe_extra_ldflags} ${EXE_LINKS_TO} %{output_to_exe}$out + + +# Executable targets + +build %{cli_exe}: link_cli %{join cli_objs} | %{library_targets} + +build %{test_exe}: link_tests %{join test_objs} | %{library_targets} + +%{if build_fuzzers} + +build fuzzers: phony %{fuzzer_bin} + +rule fuzzer_corpus + command = git clone --depth=1 https://github.com/randombit/crypto-corpus.git fuzzer_corpus + +build fuzzer_corpus: fuzzer_corpus + +rule fuzzer_corpus_zip + command = %{base_dir}/src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir} + +build fuzzer_corpus_zip: fuzzer_corpus_zip | fuzzer_corpus + +rule link_fuzzer + command = ${EXE_LINK_CMD} ${ABI_FLAGS} $in ${BUILD_DIR_LINK_PATH} ${LANG_EXE_FLAGS} ${LDFLAGS} ${EXE_LINKS_TO} %{fuzzer_lib} %{output_to_exe}$out + +%{endif} + +%{if build_examples} + +build examples: link_cli %{example_bin} | %{library_targets} + +%{endif} + + +%{if build_bogo_shim} + +build bogo_shim: link_cli botan_bogo_shim | %{library_targets} + +# BoGo shim +build %{out_dir}/botan_bogo_shim: compile_exe %{bogo_shim_src} + +%{endif} + +# Misc targets + +rule build_docs + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/build_docs.py" --build-dir="%{build_dir}" + +build %{doc_stamp_file}: build_docs + +rule clean + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/cleanup.py" --build-dir="%{build_dir}" + +rule distclean + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/cleanup.py" --build-dir="%{build_dir}" --distclean + +rule install + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/install.py" --build-dir="%{build_dir}" + +rule check + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/check.py" --build-dir="%{build_dir}" + +rule fmt + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/dev_tools/run_clang_format.py" + +rule tidy + command = "${PYTHON_EXE}" "${SCRIPTS_DIR}/dev_tools/run_clang_tidy.py" --only-changed-files + +# Target aliases + +build cli: phony %{cli_exe} + +build tests: phony %{test_exe} + +build libs: phony %{library_targets} $ +%{if symlink_shared_lib} + %{out_dir}/%{soname_base} %{out_dir}/%{soname_patch} +%{endif} + +build docs: phony %{doc_stamp_file} + +build clean: clean + +build distclean: distclean + +build install: install | %{install_targets} + +build check: check | tests + +build fmt: fmt + +build tidy: tidy + + +# Build Commands + +%{for lib_build_info} +build %{obj}: compile_lib %{src} +%{endfor} + +%{for cli_build_info} +build %{obj}: compile_exe %{src} +%{endfor} + +%{for test_build_info} +build %{obj}: compile_exe %{src} +%{endfor} + +%{for fuzzer_build_info} +build %{obj}: compile_exe %{src} + +build %{exe}: link_fuzzer %{obj} | %{library_targets} +%{endfor} + +%{for examples_build_info} +build %{obj}: compile_exe %{src} + +build %{exe}: link_cli %{obj} | %{library_targets} +%{endfor} diff --git a/src/editors/vscode/tasks.json b/src/editors/vscode/tasks.json index 433eaa7e084..b13653e8d6b 100644 --- a/src/editors/vscode/tasks.json +++ b/src/editors/vscode/tasks.json @@ -42,7 +42,7 @@ "problemMatcher": "$gcc" }, "windows": { - "command": "jom", + "command": "ninja", "problemMatcher": "$msCompile" }, "presentation": { diff --git a/src/scripts/ci/setup_gh_actions.ps1 b/src/scripts/ci/setup_gh_actions.ps1 index e975babd87f..1fd505b08f6 100644 --- a/src/scripts/ci/setup_gh_actions.ps1 +++ b/src/scripts/ci/setup_gh_actions.ps1 @@ -2,6 +2,7 @@ # # (C) 2022 Jack Lloyd # (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity +# (C) 2023 René Fischer, Rohde & Schwarz Cybersecurity # # Botan is released under the Simplified BSD License (see license.txt) @@ -11,7 +12,6 @@ param( [String]$ARCH ) -choco install -y jom choco install -y sccache # find the sccache cache location and store it in the build job's environment diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py index 0710e6e73f3..76c78d3f6e3 100755 --- a/src/scripts/ci_build.py +++ b/src/scripts/ci_build.py @@ -526,18 +526,14 @@ def validate_make_tool(make_tool, build_jobs): if make_tool == '': return validate_make_tool('make', build_jobs) - if make_tool not in ['nmake', 'jom', 'make']: + if make_tool not in ['ninja', 'nmake', 'make']: raise Exception("Don't know about %s as a make tool" % (make_tool)) - # Hack to work around jom occasionally failing to install - # https://github.com/randombit/botan/issues/3629 - if make_tool == 'jom' and not have_prog('jom'): - return ['nmake'] + if make_tool in ['make']: + return make_tool, ['-j%d' % (build_jobs), '-k'] - if make_tool in ['make', 'jom']: - return [make_tool, '-j%d' % (build_jobs)] else: - return [make_tool] + return make_tool, [] def main(args=None): """ @@ -650,13 +646,17 @@ def main(args=None): options.pkcs11_lib, options.use_gdb, options.disable_werror, options.extra_cxxflags, options.disabled_tests) - cmds.append([py_interp, os.path.join(root_dir, 'configure.py')] + config_flags) + make_tool, make_opts = validate_make_tool(options.make_tool, options.build_jobs) - make_cmd = validate_make_tool(options.make_tool, options.build_jobs) - if build_dir != '.': - make_cmd = ['indir:%s' % build_dir] + make_cmd + cmds.append([py_interp, + os.path.join(root_dir, 'configure.py')] + + ['--build-tool=' + make_tool] + + config_flags) - make_cmd += ['-k'] + make_cmd = [make_tool] + make_opts + + if build_dir != '.': + make_cmd = ['indir:%s' % build_dir] + [make_tool] + make_opts if target == 'docs': cmds.append(make_cmd + ['docs'])