#2: use YAML reference features in config

ci/build-docker-image

@@ -56,18 +56,17 @@ class DockerBuilder:
 
         image_setup = setup.get(image.get('setup'))
 
-        # Add compiler packages
-        compiler = image_setup.get('compiler')
+        env = image_setup.get('env')
         args = {
             # General
             "ARCH": image.get('arch'),
             "BASE": image.get('base'),
             "SETUP_ID": image.get('setup'),
-            # Compiler (C, C++, Fortran, coverage tool)
-            "CC": compiler.get('cc', '') if compiler is not None else '',
-            "CXX": compiler.get('cxx', '') if compiler is not None else '',
-            "FC": compiler.get('fc', '') if compiler is not None else '',
-            "GCOV": compiler.get('gcov', '') if compiler is not None else ''
+            # Environment
+            "CC": env.get('CC', ''),
+            "CXX": env.get('CXX', ''),
+            "FC": env.get('FC', ''),
+            "GCOV": env.get('GCOV', '')
         }
 
         space = ' '

ci/build-setup

@@ -12,8 +12,8 @@ class SetupBuilder:
 
     def __instructions(self, dep_id, args: list) -> List[str]:
         """ Generate shell instructions to setup a dependency"""
-
         call_args = []
+        # repeat instructions if args is an array of array
         if args is not None and len(args) > 0:
             if isinstance(args[0], list):
                 instructions = []
@@ -37,29 +37,11 @@ class SetupBuilder:
         config = resolve_conf(copy.deepcopy(raw_config))
 
         setup = config.get("setup")
-
-        for (env_id, environment) in setup.items():
-            # Add compiler packages
-            compiler = environment.get('compiler')
-            packages = copy.deepcopy(environment.get("deps", {}).get("packages", []))
-            compiler_packages = compiler.get("packages",
-                                            [ c for c in [compiler.get("cc"), compiler.get("cxx")]
-                                                            if c is not None])
-            if compiler_packages is None or compiler_packages is False:
-                compiler_packages = []
-
-            for compiler_package in compiler_packages:
-                if packages is None:
-                    packages = []
-                packages.append(compiler_package)
-
-            if len(packages) > 0:
-                environment["deps"]["packages"] = packages
-
+        for (setup_id, setup_config) in setup.items():
             # generate install instructions and install dependencies commands
             instructions = []
             downloads = []
-            for (dep_id, args) in environment.get("deps").items():
+            for (dep_id, args) in setup_config.get("deps").items():
                 downloads.append(f"wget $SCRIPTS_DEPS_URL/{dep_id}.sh")
                 instructions.extend(self.__instructions(dep_id, args))
 
@@ -70,11 +52,11 @@ class SetupBuilder:
                 encoding="utf-8"
             ) as file:
                 setup_script = file.read()
-            setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', environment.get("label"))
+            setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', setup_config.get("label"))
             setup_script = setup_script.replace('%DEPS_DOWNLOAD%', '\n'.join(downloads))
             setup_script = setup_script.replace('%DEPS_INSTALL%', '\n'.join(instructions))
 
-            setup_filename = f"setup-{env_id}.sh"
+            setup_filename = f"setup-{setup_id}.sh"
             setup_filepath = os.path.join(os.path.dirname(__file__),
                                           'shared', 'scripts', setup_filename)
 

ci/config.yaml

@@ -13,56 +13,76 @@
 # THIS IS A LOT OF WORK BUT MIGHT BE VERY USEFUL
 
 defaults:
-  apt-packages: [
+  # default packages
+  apt-packages: &apt-packages [
     curl, jq, less, libomp5, libunwind-dev make-guile, ninja-build,
     valgrind, zlib1g, zlib1g-dev, ccache, python3
   ]
-  apk-packages: [
+  apk-packages: &apk-packages [
     alpine-sdk, autoconf, automake, binutils-dev, ccache, cmake, dpkg,
     libdwarf-dev, libunwind-dev, libtool, linux-headers, m4, make, ninja, zlib,
     zlib-dev
   ]
-  brew-packages: [ ccache, coreutils ]
-  docker-repository: lifflander1/vt
-  pip-packages: "nanobind yaml setuptools"
-  conda-py3.all:
-      - ["3.8", "@defaults.pip-packages"]
-      - ["3.9", "@defaults.pip-packages"]
-      - ["3.10", "@defaults.pip-packages"]
-      - ["3.11", "@defaults.pip-packages"]
-      - ["3.12", "@defaults.pip-packages"]
+  brew-packages: &brew-packages [ ccache, coreutils ]
+  pip-packages: &pip-packages "nanobind yaml setuptools"
 
-compilers:
-  clang-11: { type: clang, cc: clang-11, cxx: clang++-11, fc: flang-11, gcov: llvm-gcov }
-  gcc-9: { type: gnu, cc: gcc-9, cxx: g++-9, fc: gfortran-9, gcov: gcov-9 }
-  gcc-11: { type: gnu, cc: gcc-11, cxx: g++-11, fc: gfortran-11, gcov: gcov-11 }
-  gcc-12: { type: gnu, cc: gcc-12, cxx: g++-12, fc: gfortran-12, gcov: gcov-12 }
-  gcc-13: { type: gnu, cc: gcc-13, cxx: g++-13, fc: gfortran-13, gcov: gcov-13 }
+  # default arguments for dependency installation scripts
+  cmake-args: &cmake-args '3.23.4'
+  openmpi-args: &openmpi-args ['v5.0', '5.0.4']
+  zoltan-args: &zoltan-args [-j4', '/trilinos-install']
+  mpich-args: &mpich-args ['4.0.2', '-j4']
+  vtk-args: &vtk-args '9.3.1'
+
+  # other
+  docker-repository: &docker-repository lifflander1/vt 
+  conda-py3-all: &conda-py3-all [
+    ["3.8", *pip-packages], ["3.9", *pip-packages], ["3.10", *pip-packages],
+    ["3.11", *pip-packages], ["3.12", *pip-packages]
+  ]
+
+
+# compilers:
+#   clang-11: &clang-11 { type: clang, cc: clang-11, cxx: clang++-11, fc: flang-11, gcov: llvm-gcov }
+#   gcc-9: &gcc-9 { type: gnu, cc: gcc-9, cxx: g++-9, fc: gfortran-9, gcov: gcov-9 }
+#   gcc-11: &gcc-11 { type: gnu, cc: gcc-11, cxx: g++-11, fc: gfortran-11, gcov: gcov-11 }
+#   gcc-12: &gcc-12 { type: gnu, cc: gcc-12, cxx: g++-12, fc: gfortran-12, gcov: gcov-12 }
+#   gcc-13: &gcc-13 { type: gnu, cc: gcc-13, cxx: g++-13, fc: gfortran-13, gcov: gcov-13 }
 
 setup:
   macos-14-clang-14:
     label: macos-14-arm64, clang-14
-    compiler: { type: clang, cc: clang, cxx: clang++, gcov: llvm-gcov, packages: false }
+    env:
+      CC: clang
+      CXX: clang++
+      GCOV: llvm-gcov
     deps:
-      packages: "@defaults.brew-packages"
+      packages: [ *brew-packages, clang, clang++ ]
 
   amd64-alpine-clang-13-clang-13-cpp:
     label: alpine-3.16, clang-13
-    compiler: { type: clang, cc: clang, cxx: clang++, gcov: llvm-gcov, packages: [ clang, clang-dev ] }
+    env:
+      CC: mpicc
+      CXX: mpicxx
+      GCOV: llvm-gcov
     deps:
-      packages: "@defaults.apk-packages"
+      packages: [ *apk-packages, clang, clang-dev ]
+      mpich: ['3.3.2', '-j4', 'clang', 'clang++']
+
 
   #vt
   ubuntu-20.04-gcc-9-12.2.0-cpp:
     label: vt, gcc-9, ubuntu, mpich, zoltan
-    compiler: "@compilers.gcc-9"
+    env:
+      CC: gcc-9
+      CXX: g++-9
+      GCOV: gcov
     # deps and versions to find in vt
     deps:
-      packages: "@defaults.apt-packages"
-      cmake: ['3.23.4']
-      openmpi: ['v5.0', '5.0.4']
-      zoltan: [-j4', '/trilinos-install']
-      mpich: ['4.0.2', '-j4']
+      packages: [ *apt-packages, gcc-9, g++-9 ]
+      cmake: *cmake-args
+      openmpi: *openmpi-args
+      zoltan: *zoltan-args
+      mpich: *mpich-args
 
   # TODO: add the following
   # amd64-ubuntu-20.04-gcc-9-11.2.2-cpp:
@@ -86,60 +106,72 @@ setup:
   # vt-tv
   ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all:
     label: vt-tv, gcc-13, ubuntu, vtk 9.3.1, py3[8-12]
-    compiler: "@compilers.gcc-13"
+    env:
+      CC: gcc-13
+      CXX: g++-13
+      GCOV: gcov
     deps:
-      packages: "@defaults.apt-packages"
+      packages: [ *apt-packages, gcc-13, g++-13 ]
       mesa: ~
       conda: ~
-      conda-python-env: "@defaults.conda-py3.all"
-      cmake: ['3.30.3']
-      vtk: [ '9.3.1' ]
+      conda-python-env: *conda-py3-all
+      cmake: *cmake-args
+      vtk: *vtk-args
 
   ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all:
     label: vt-tv, gcc-12, ubuntu, vtk 9.3.0, py3[8-12]
-    compiler: "@compilers.gcc-12"
+    env:
+      CC: gcc-12
+      CXX: g++-12
+      GCOV: gcov
     deps:
-      packages: "@defaults.apt-packages"
+      packages: [ *apt-packages, gcc-12, g++-12 ]
       mesa: ~
       conda: ~
-      conda-python-env: "@defaults.conda-py3.all"
-      cmake: ['3.30.3']
+      conda-python-env: *conda-py3-all
+      cmake: *cmake-args
       vtk: [ '9.3.0' ]
 
   ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all:
     label: vt-tv, clang-11, ubuntu, vtk 9.2.2, py3[8-12]
-    compiler: "@compilers.clang-11"
+    env:
+      CC: gcc-11
+      CXX: g++-11
+      GCOV: gcov
     deps:
-      packages: "@defaults.apt-packages"
+      packages: [*apt-packages, gcc-11, g++-11 ]
       mesa: ~
       conda: ~
-      conda-python-env: "@defaults.conda-py3.all"
-      cmake: ['3.30.3']
+      conda-python-env: *conda-py3-all
+      cmake: *cmake-args
       vtk: [ '9.2.2' ]
 
   ubuntu-22.04-gcc-11-vtk-9.2.2-py-3.all:
     label: vt-tv, gcc-11, ubuntu, vtk 9.2.2, py3[8-12]
-    compiler: "@compilers.gcc-11"
+    env:
+      CC: gcc-11
+      CXX: g++-11
+      GCOV: gcov
     deps:
-      packages: "@defaults.apt-packages"
+      packages: [*apt-packages, gcc-11, g++-11 ]
       mesa: ~
       conda: ~
-      conda-python-env: "@defaults.conda-py3.all"
-      cmake: ['3.30.3']
+      conda-python-env: *conda-py3-all
+      cmake: *cmake-args
       vtk: [ '9.2.2' ]
 
 images:
   # vt
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: amd64-ubuntu-20.04-gcc-9-12.2.0-cpp
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64
     base: ubuntu:20.04
     setup: ubuntu-20.04-gcc-9-12.2.0-cpp
 
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: amd64-alpine-clang-13-clang-13-cpp
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64
@@ -167,28 +199,28 @@ images:
 
   # vt-tv
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: ubuntu_24.04-gcc_13-vtk_9.3.1-py_3.all
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64
     base: ubuntu:24.04
     setup: ubuntu-24.04-gcc-13-vtk-9.3.1-py-3.all
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: ubuntu_22.04-gcc_12-vtk_9.3.0-py_3.all
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64
     base: ubuntu:22.04
     setup: ubuntu-22.04-gcc-12-vtk-9.3.0-py-3.all
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: ubuntu_22.04-clang_11-vtk_9.2.2-py_3.all
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64
     base: ubuntu:22.04
     setup: ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all
   -
-    repository: "@defaults.docker-repository"
+    repository: *docker-repository
     tag: ubuntu_22.04-gcc_11-vtk_9.2.2-py_3.all
     dockerfile: ubuntu-cpp-base.dockerfile
     arch: amd64

ci/shared/scripts/deps/mpich.sh

@@ -12,7 +12,8 @@ mpich_version=$1
 mpich_name="mpich-${mpich_version}"
 mpich_tar_name="${mpich_name}.tar.gz"
 make_flags="$2"
-installation_prefix="${3-}"
+mpich_cc="${3:-$CC}"
+mpich_cxx="${4:-$CXX}"
 
 echo "${mpich_version}"
 echo "${mpich_name}"
@@ -23,6 +24,10 @@ wget http://www.mpich.org/static/downloads/${mpich_version}/${mpich_tar_name}
 tar xzf ${mpich_tar_name}
 rm ${mpich_tar_name}
 cd ${mpich_name}
+
+export CC=$mpich_cc
+export CXX=$mpich_cxx
+
 ./configure \
     --enable-static=false \
     --enable-alloca=true \
@@ -36,4 +41,7 @@ cd ${mpich_name}
 make ${make_flags}
 make install
 cd -
-rm -rf ${mpich_name}
+rm -rf ${mpich_name}
+
+which mpicc
+which mpicxx

ci/shared/scripts/setup-amd64-alpine-clang-13-clang-13-cpp.sh

@@ -68,10 +68,12 @@ mkdir -p $SCRIPTS_INSTALL_DIR/deps
 # 1. Download dependency installation script
 cd $SCRIPTS_INSTALL_DIR/deps
 wget $SCRIPTS_DEPS_URL/packages.sh
+wget $SCRIPTS_DEPS_URL/mpich.sh
 # 2. Install dependency
 chmod u+x *.sh
 ls -l
 ./packages.sh "alpine-sdk" "autoconf" "automake" "binutils-dev" "ccache" "cmake" "dpkg" "libdwarf-dev" "libunwind-dev" "libtool" "linux-headers" "m4" "make" "ninja" "zlib" "zlib-dev" "clang" "clang-dev"
+./mpich.sh "3.3.2" "-j4" "clang" "clang++"
 
 # Remove install scripts
 rm -rf $SCRIPTS_INSTALL_DIR

ci/shared/scripts/setup-macos-14-clang-14.sh

@@ -71,7 +71,7 @@ wget $SCRIPTS_DEPS_URL/packages.sh
 # 2. Install dependency
 chmod u+x *.sh
 ls -l
-./packages.sh "ccache" "coreutils"
+./packages.sh "ccache" "coreutils" "clang" "clang++"
 
 # Remove install scripts
 rm -rf $SCRIPTS_INSTALL_DIR

ci/shared/scripts/setup-ubuntu-22.04-clang-11-vtk-9.2.2-py-3.all.sh

@@ -76,15 +76,15 @@ wget $SCRIPTS_DEPS_URL/vtk.sh
 # 2. Install dependency
 chmod u+x *.sh
 ls -l
-./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "clang-11" "clang++-11"
-./mesa.sh
-./conda.sh
+./packages.sh "curl" "jq" "less" "libomp5" "libunwind-dev make-guile" "ninja-build" "valgrind" "zlib1g" "zlib1g-dev" "ccache" "python3" "gcc-11" "g++-11"
+./mesa.sh "None"
+./conda.sh "None"
 ./conda-python-env.sh "3.8" "nanobind yaml setuptools"
 ./conda-python-env.sh "3.9" "nanobind yaml setuptools"
 ./conda-python-env.sh "3.10" "nanobind yaml setuptools"
 ./conda-python-env.sh "3.11" "nanobind yaml setuptools"
 ./conda-python-env.sh "3.12" "nanobind yaml setuptools"
-./cmake.sh "3.30.3"
+./cmake.sh "3.23.4"
 ./vtk.sh "9.2.2"
 
 # Remove install scripts