diff --git a/ci/build-docker-image b/ci/build-docker-image index 51a74d31..4ed1f5ca 100755 --- a/ci/build-docker-image +++ b/ci/build-docker-image @@ -56,18 +56,17 @@ class DockerBuilder: image_setup = setup.get(image.get('setup')) - # Add compiler packages - compiler = image_setup.get('compiler') + env = image_setup.get('env') args = { # General "ARCH": image.get('arch'), "BASE": image.get('base'), "SETUP_ID": image.get('setup'), - # Compiler (C, C++, Fortran, coverage tool) - "CC": compiler.get('cc', '') if compiler is not None else '', - "CXX": compiler.get('cxx', '') if compiler is not None else '', - "FC": compiler.get('fc', '') if compiler is not None else '', - "GCOV": compiler.get('gcov', '') if compiler is not None else '' + # Environment + "CC": env.get('CC', ''), + "CXX": env.get('CXX', ''), + "FC": env.get('FC', ''), + "GCOV": env.get('GCOV', '') } space = ' ' diff --git a/ci/build-setup b/ci/build-setup index bfe32ca0..567c4e41 100755 --- a/ci/build-setup +++ b/ci/build-setup @@ -12,8 +12,8 @@ class SetupBuilder: def __instructions(self, dep_id, args: list) -> List[str]: """ Generate shell instructions to setup a dependency""" - call_args = [] + # repeat instructions if args is an array of array if args is not None and len(args) > 0: if isinstance(args[0], list): instructions = [] @@ -37,29 +37,11 @@ class SetupBuilder: config = resolve_conf(copy.deepcopy(raw_config)) setup = config.get("setup") - - for (env_id, environment) in setup.items(): - # Add compiler packages - compiler = environment.get('compiler') - packages = copy.deepcopy(environment.get("deps", {}).get("packages", [])) - compiler_packages = compiler.get("packages", - [ c for c in [compiler.get("cc"), compiler.get("cxx")] - if c is not None]) - if compiler_packages is None or compiler_packages is False: - compiler_packages = [] - - for compiler_package in compiler_packages: - if packages is None: - packages = [] - packages.append(compiler_package) - - if len(packages) > 0: - environment["deps"]["packages"] = packages - + for (setup_id, setup_config) in setup.items(): # generate install instructions and install dependencies commands instructions = [] downloads = [] - for (dep_id, args) in environment.get("deps").items(): + for (dep_id, args) in setup_config.get("deps").items(): downloads.append(f"wget $SCRIPTS_DEPS_URL/{dep_id}.sh") instructions.extend(self.__instructions(dep_id, args)) @@ -70,11 +52,11 @@ class SetupBuilder: encoding="utf-8" ) as file: setup_script = file.read() - setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', environment.get("label")) + setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', setup_config.get("label")) setup_script = setup_script.replace('%DEPS_DOWNLOAD%', '\n'.join(downloads)) setup_script = setup_script.replace('%DEPS_INSTALL%', '\n'.join(instructions)) - setup_filename = f"setup-{env_id}.sh" + setup_filename = f"setup-{setup_id}.sh" setup_filepath = os.path.join(os.path.dirname(__file__), 'shared', 'scripts', setup_filename) diff --git a/ci/config.yaml b/ci/config.yaml index dc84b842..ed6852b0 100644 --- a/ci/config.yaml +++ b/ci/config.yaml @@ -13,56 +13,76 @@ # THIS IS A LOT OF WORK BUT MIGHT BE VERY USEFUL defaults: - apt-packages: [ + # default packages + apt-packages: &apt-packages [ curl, jq, less, libomp5, libunwind-dev make-guile, ninja-build, valgrind, zlib1g, zlib1g-dev, ccache, python3 ] - apk-packages: [ + apk-packages: &apk-packages [ alpine-sdk, autoconf, automake, binutils-dev, ccache, cmake, dpkg, libdwarf-dev, libunwind-dev, libtool, linux-headers, m4, make, ninja, zlib, zlib-dev ] - brew-packages: [ ccache, coreutils ] - docker-repository: lifflander1/vt - pip-packages: "nanobind yaml setuptools" - conda-py3.all: - - ["3.8", "@defaults.pip-packages"] - - ["3.9", "@defaults.pip-packages"] - - ["3.10", "@defaults.pip-packages"] - - ["3.11", "@defaults.pip-packages"] - - ["3.12", "@defaults.pip-packages"] + brew-packages: &brew-packages [ ccache, coreutils ] + pip-packages: &pip-packages "nanobind yaml setuptools" -compilers: - clang-11: { type: clang, cc: clang-11, cxx: clang++-11, fc: flang-11, gcov: llvm-gcov } - gcc-9: { type: gnu, cc: gcc-9, cxx: g++-9, fc: gfortran-9, gcov: gcov-9 } - gcc-11: { type: gnu, cc: gcc-11, cxx: g++-11, fc: gfortran-11, gcov: gcov-11 } - gcc-12: { type: gnu, cc: gcc-12, cxx: g++-12, fc: gfortran-12, gcov: gcov-12 } - gcc-13: { type: gnu, cc: gcc-13, cxx: g++-13, fc: gfortran-13, gcov: gcov-13 } + # default arguments for dependency installation scripts + cmake-args: &cmake-args '3.23.4' + openmpi-args: &openmpi-args ['v5.0', '5.0.4'] + zoltan-args: &zoltan-args [-j4', '/trilinos-install'] + mpich-args: &mpich-args ['4.0.2', '-j4'] + vtk-args: &vtk-args '9.3.1' + + # other + docker-repository: &docker-repository lifflander1/vt + conda-py3-all: &conda-py3-all [ + ["3.8", *pip-packages], ["3.9", *pip-packages], ["3.10", *pip-packages], + ["3.11", *pip-packages], ["3.12", *pip-packages] + ] + + +# compilers: +# clang-11: &clang-11 { type: clang, cc: clang-11, cxx: clang++-11, fc: flang-11, gcov: llvm-gcov } +# gcc-9: &gcc-9 { type: gnu, cc: gcc-9, cxx: g++-9, fc: gfortran-9, gcov: gcov-9 } +# gcc-11: &gcc-11 { type: gnu, cc: gcc-11, cxx: g++-11, fc: gfortran-11, gcov: gcov-11 } +# gcc-12: &gcc-12 { type: gnu, cc: gcc-12, cxx: g++-12, fc: gfortran-12, gcov: gcov-12 } +# gcc-13: &gcc-13 { type: gnu, cc: gcc-13, cxx: g++-13, fc: gfortran-13, gcov: gcov-13 } setup: macos-14-clang-14: label: macos-14-arm64, clang-14 - compiler: { type: clang, cc: clang, cxx: clang++, gcov: llvm-gcov, packages: false } + env: + CC: clang + CXX: clang++ + GCOV: llvm-gcov deps: - packages: "@defaults.brew-packages" + packages: [ *brew-packages, clang, clang++ ] amd64-alpine-clang-13-clang-13-cpp: label: alpine-3.16, clang-13 - compiler: { type: clang, cc: clang, cxx: clang++, gcov: llvm-gcov, packages: [ clang, clang-dev ] } + env: + CC: mpicc + CXX: mpicxx + GCOV: llvm-gcov deps: - packages: "@defaults.apk-packages" + packages: [ *apk-packages, clang, clang-dev ] + mpich: ['3.3.2', '-j4'] + #vt ubuntu-20.04-gcc-9-12.2.0-cpp: label: vt, gcc-9, ubuntu, mpich, zoltan - compiler: "@compilers.gcc-9" + env: + CC: gcc-9 + CXX: g++-9 + GCOV: gcov # deps and versions to find in vt deps: - packages: "@defaults.apt-packages" - cmake: ['3.23.4'] - openmpi: ['v5.0', '5.0.4'] - zoltan: [-j4', '/trilinos-install'] - mpich: ['4.0.2', '-j4'] + packages: [ *apt-packages, gcc-9, g++-9 ] + cmake: *cmake-args + openmpi: *openmpi-args + zoltan: *zoltan-args + mpich: *mpich-args # TODO: add the following # amd64-ubuntu-20.04-gcc-9-11.2.2-cpp: @@ -86,52 +106,64 @@ setup: # vt-tv ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all: label: vt-tv, gcc-13, ubuntu, vtk 9.3.1, py3[8-12] - compiler: "@compilers.gcc-13" + env: + CC: gcc-13 + CXX: g++-13 + GCOV: gcov deps: - packages: "@defaults.apt-packages" + packages: [ *apt-packages, gcc-13, g++-13 ] mesa: ~ conda: ~ - conda-python-env: "@defaults.conda-py3.all" - cmake: ['3.30.3'] - vtk: [ '9.3.1' ] + conda-python-env: *conda-py3-all + cmake: *cmake-args + vtk: *vtk-args ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all: label: vt-tv, gcc-12, ubuntu, vtk 9.3.0, py3[8-12] - compiler: "@compilers.gcc-12" + env: + CC: gcc-12 + CXX: g++-12 + GCOV: gcov deps: - packages: "@defaults.apt-packages" + packages: [ *apt-packages, gcc-12, g++-12 ] mesa: ~ conda: ~ - conda-python-env: "@defaults.conda-py3.all" - cmake: ['3.30.3'] + conda-python-env: *conda-py3-all + cmake: *cmake-args vtk: [ '9.3.0' ] ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all: label: vt-tv, clang-11, ubuntu, vtk 9.2.2, py3[8-12] - compiler: "@compilers.clang-11" + env: + CC: gcc-11 + CXX: g++-11 + GCOV: gcov deps: - packages: "@defaults.apt-packages" + packages: [*apt-packages, gcc-11, g++-11 ] mesa: ~ conda: ~ - conda-python-env: "@defaults.conda-py3.all" - cmake: ['3.30.3'] + conda-python-env: *conda-py3-all + cmake: *cmake-args vtk: [ '9.2.2' ] ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all: label: vt-tv, gcc-11, ubuntu, vtk 9.2.2, py3[8-12] - compiler: "@compilers.gcc-11" + env: + CC: gcc-11 + CXX: g++-11 + GCOV: gcov deps: - packages: "@defaults.apt-packages" + packages: [*apt-packages, gcc-11, g++-11 ] mesa: ~ conda: ~ - conda-python-env: "@defaults.conda-py3.all" - cmake: ['3.30.3'] + conda-python-env: *conda-py3-all + cmake: *cmake-args vtk: [ '9.2.2' ] images: # vt - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: amd64-ubuntu-20.04-gcc-9-12.2.0-cpp dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 @@ -139,7 +171,7 @@ images: setup: ubuntu-20.04-gcc-9-12.2.0-cpp - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: amd64-alpine-clang-13-clang-13-cpp dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 @@ -167,28 +199,28 @@ images: # vt-tv - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: ubuntu_24.04-gcc_13-vtk_9.3.1-py_3.all dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 base: ubuntu:24.04 setup: ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: ubuntu_22.04-gcc_12-vtk_9.3.0-py_3.all dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 base: ubuntu:22.04 setup: ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: ubuntu_22.04-clang_11-vtk_9.2.2-py_3.all dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 base: ubuntu:22.04 setup: ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all - - repository: "@defaults.docker-repository" + repository: *docker-repository tag: ubuntu_22.04-gcc_11-vtk_9.2.2-py_3.all dockerfile: ubuntu-cpp-base.dockerfile arch: amd64 diff --git a/ci/shared/scripts/deps/mpich.sh b/ci/shared/scripts/deps/mpich.sh index 06cdcab7..1a8b2e22 100644 --- a/ci/shared/scripts/deps/mpich.sh +++ b/ci/shared/scripts/deps/mpich.sh @@ -12,7 +12,8 @@ mpich_version=$1 mpich_name="mpich-${mpich_version}" mpich_tar_name="${mpich_name}.tar.gz" make_flags="$2" -installation_prefix="${3-}" +mpich_cc="${3:-$CC}" +mpich_cxx="${4:-$CXX}" echo "${mpich_version}" echo "${mpich_name}" @@ -23,6 +24,10 @@ wget http://www.mpich.org/static/downloads/${mpich_version}/${mpich_tar_name} tar xzf ${mpich_tar_name} rm ${mpich_tar_name} cd ${mpich_name} + +export CC=$mpich_cc +export CXX=$mpich_cxx + ./configure \ --enable-static=false \ --enable-alloca=true \ @@ -36,4 +41,7 @@ cd ${mpich_name} make ${make_flags} make install cd - -rm -rf ${mpich_name} \ No newline at end of file +rm -rf ${mpich_name} + +which mpicc +which mpicxx \ No newline at end of file diff --git a/ci/shared/scripts/setup-amd64-alpine-clang-13-clang-13-cpp.sh b/ci/shared/scripts/setup-amd64-alpine-clang-13-clang-13-cpp.sh index 276c9337..a4feed7f 100644 --- a/ci/shared/scripts/setup-amd64-alpine-clang-13-clang-13-cpp.sh +++ b/ci/shared/scripts/setup-amd64-alpine-clang-13-clang-13-cpp.sh @@ -68,10 +68,12 @@ mkdir -p $SCRIPTS_INSTALL_DIR/deps # 1. Download dependency installation script cd $SCRIPTS_INSTALL_DIR/deps wget $SCRIPTS_DEPS_URL/packages.sh +wget $SCRIPTS_DEPS_URL/mpich.sh # 2. Install dependency chmod u+x *.sh ls -l ./packages.sh "alpine-sdk" "autoconf" "automake" "binutils-dev" "ccache" "cmake" "dpkg" "libdwarf-dev" "libunwind-dev" "libtool" "linux-headers" "m4" "make" "ninja" "zlib" "zlib-dev" "clang" "clang-dev" +./mpich.sh "3.3.2" "CC=clang CXX=clang++ -j4" # Remove install scripts rm -rf $SCRIPTS_INSTALL_DIR diff --git a/ci/shared/scripts/setup-macos-14-clang-14.sh b/ci/shared/scripts/setup-macos-14-clang-14.sh index 9cd8a2af..66fa4f24 100644 --- a/ci/shared/scripts/setup-macos-14-clang-14.sh +++ b/ci/shared/scripts/setup-macos-14-clang-14.sh @@ -71,7 +71,7 @@ wget $SCRIPTS_DEPS_URL/packages.sh # 2. Install dependency chmod u+x *.sh ls -l -./packages.sh "ccache" "coreutils" +./packages.sh "ccache" "coreutils" "clang" "clang++" # Remove install scripts rm -rf $SCRIPTS_INSTALL_DIR diff --git a/ci/shared/scripts/setup-ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all.sh b/ci/shared/scripts/setup-ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all.sh index cfebc1cb..bfcae0c7 100644 --- a/ci/shared/scripts/setup-ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all.sh +++ b/ci/shared/scripts/setup-ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all.sh @@ -76,15 +76,15 @@ wget $SCRIPTS_DEPS_URL/vtk.sh # 2. Install dependency chmod u+x *.sh ls -l -./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "clang-11" "clang++-11" -./mesa.sh -./conda.sh +./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "gcc-11" "g++-11" +./mesa.sh "None" +./conda.sh "None" ./conda-python-env.sh "3.8" "nanobind yaml setuptools" ./conda-python-env.sh "3.9" "nanobind yaml setuptools" ./conda-python-env.sh "3.10" "nanobind yaml setuptools" ./conda-python-env.sh "3.11" "nanobind yaml setuptools" ./conda-python-env.sh "3.12" "nanobind yaml setuptools" -./cmake.sh "3.30.3" +./cmake.sh "3.23.4" ./vtk.sh "9.2.2" # Remove install scripts diff --git a/ci/shared/scripts/setup-ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all.sh b/ci/shared/scripts/setup-ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all.sh index 28af8c65..c8095491 100644 --- a/ci/shared/scripts/setup-ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all.sh +++ b/ci/shared/scripts/setup-ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all.sh @@ -77,14 +77,14 @@ wget $SCRIPTS_DEPS_URL/vtk.sh chmod u+x *.sh ls -l ./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "gcc-11" "g++-11" -./mesa.sh -./conda.sh +./mesa.sh "None" +./conda.sh "None" ./conda-python-env.sh "3.8" "nanobind yaml setuptools" ./conda-python-env.sh "3.9" "nanobind yaml setuptools" ./conda-python-env.sh "3.10" "nanobind yaml setuptools" ./conda-python-env.sh "3.11" "nanobind yaml setuptools" ./conda-python-env.sh "3.12" "nanobind yaml setuptools" -./cmake.sh "3.30.3" +./cmake.sh "3.23.4" ./vtk.sh "9.2.2" # Remove install scripts diff --git a/ci/shared/scripts/setup-ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all.sh b/ci/shared/scripts/setup-ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all.sh index e08cc8aa..95290668 100644 --- a/ci/shared/scripts/setup-ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all.sh +++ b/ci/shared/scripts/setup-ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all.sh @@ -77,14 +77,14 @@ wget $SCRIPTS_DEPS_URL/vtk.sh chmod u+x *.sh ls -l ./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "gcc-12" "g++-12" -./mesa.sh -./conda.sh +./mesa.sh "None" +./conda.sh "None" ./conda-python-env.sh "3.8" "nanobind yaml setuptools" ./conda-python-env.sh "3.9" "nanobind yaml setuptools" ./conda-python-env.sh "3.10" "nanobind yaml setuptools" ./conda-python-env.sh "3.11" "nanobind yaml setuptools" ./conda-python-env.sh "3.12" "nanobind yaml setuptools" -./cmake.sh "3.30.3" +./cmake.sh "3.23.4" ./vtk.sh "9.3.0" # Remove install scripts diff --git a/ci/shared/scripts/setup-ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all.sh b/ci/shared/scripts/setup-ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all.sh index 79bd0cff..b5622a05 100644 --- a/ci/shared/scripts/setup-ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all.sh +++ b/ci/shared/scripts/setup-ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all.sh @@ -77,14 +77,14 @@ wget $SCRIPTS_DEPS_URL/vtk.sh chmod u+x *.sh ls -l ./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "gcc-13" "g++-13" -./mesa.sh -./conda.sh +./mesa.sh "None" +./conda.sh "None" ./conda-python-env.sh "3.8" "nanobind yaml setuptools" ./conda-python-env.sh "3.9" "nanobind yaml setuptools" ./conda-python-env.sh "3.10" "nanobind yaml setuptools" ./conda-python-env.sh "3.11" "nanobind yaml setuptools" ./conda-python-env.sh "3.12" "nanobind yaml setuptools" -./cmake.sh "3.30.3" +./cmake.sh "3.23.4" ./vtk.sh "9.3.1" # Remove install scripts diff --git a/ci/util.py b/ci/util.py index d8ab1603..a46e293f 100644 --- a/ci/util.py +++ b/ci/util.py @@ -3,48 +3,11 @@ from typing import Union -def resolve_item_params(item, root: Union[dict,list] = None): - if isinstance(item, str) and item.startswith('@'): - ref_name = item[1:] - ref_path = ref_name.split(".", 1) - ref_group_key = "parameters" - if len(ref_path) == 2 and ref_path[0] in root.keys(): - ref_group_key = ref_path[0] - ref_name = ref_path[1] - ref_group = root.get(ref_group_key) - - if ref_group is None or not ref_name in ref_group.keys() or ref_group is None: - raise ValueError(f"Reference {item} not found !") - - return ref_group.get(ref_name) - return item - -def resolve_params(data: Union[dict,list], root: Union[dict,list] = None) -> dict: - """Replace parameters variables '@parameter_name' by the corresponding - parameter value recursively. - """ - - if root is None: - root = data - - if isinstance(data, list): - for i, item in enumerate(data): - if isinstance(item, dict) or isinstance(item, list): - data[i] = resolve_params(item, root) - else: - data[i] = resolve_item_params(item, root) - else: - if isinstance(data, dict): - for item in data: - if isinstance(data[item], dict) or isinstance(data[item], list): - data[item] = resolve_params(data[item], root) - else: - data[item] = resolve_item_params(data[item], root) - return data - def resolve_conf(config: Union[dict,list]) -> dict: - config = resolve_params(config) - + """Update configuration to ease its processing: + - Turn images list into a dict (images indexed by `repo:tag`) + - Add some required missing keys with default values + """ if config.get("images") is None: config["images"] = [] @@ -58,6 +21,24 @@ def resolve_conf(config: Union[dict,list]) -> dict: if setup.get("deps") is None: setup["deps"] = {} + for dep_id, args in setup["deps"].items(): + # no args + if args is None: + setup["deps"][dep_id] = [] + # single arg + if not isinstance(args, list): + setup["deps"][dep_id] = [ args ] + + # flatten package dependencies + if setup["deps"].get("packages") is not None: + flattened_args = [] + for (_, arg) in enumerate(setup["deps"].get("packages")): + if isinstance(arg, list): + flattened_args.extend(arg) + else: + flattened_args.append(arg) + setup["deps"]["packages"] = flattened_args + # Index Docker images by their full name config["images"] = dict((image.get("repository") + ":" + image.get("tag"), image) for image in config.get("images"))