#2: rename ci python scripts

.github/workflows/build-docker-images.yml

@@ -42,4 +42,4 @@ jobs:
       - run: pip install pyyaml
       - name: Docker image build
         run: |
-          python ci/build-docker-image ${{ matrix.runner.image }}
+          python ci/build-docker-image.py ${{ matrix.runner.image }}

ci/build-docker-image.py

@@ -0,0 +1,85 @@
+import copy
+import os
+import sys
+
+from util import resolve_conf
+import yaml
+
+class DockerBuilder:
+    """Dockerfile generator class"""
+
+    def build(self, args: list):
+        """Build an image using a given docker configuration fro the config file"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+
+        config = resolve_conf(copy.deepcopy(raw_config))
+        images = config.get("images")
+        setup = config.get("setup")
+
+        image_tag = None
+        if len(args) > 0:
+            image_tag = args[0]
+            if image_tag not in images.keys():
+                print(f"[error] Image not found {image_tag}.\n"
+                      f"Available images:{(os.linesep + '- ')}"
+                      f"{(os.linesep + '- ') . join(images.keys())}")
+                raise SystemExit(1)
+        else:
+            # Step 1: list platforms and their configurations
+            choices = {k: v for k, v in enumerate(images.keys())}
+            print("Choose image: ")
+            for i in choices:
+                image = images.get(choices[i])
+                setup_id = image.get('setup')
+                current_setup = setup.get(setup_id)
+                if current_setup is None:
+                    raise RuntimeError(f"Invalid setup {setup_id}")
+                lbl = current_setup.get('label', image.get('setup'))
+                print(
+                    f"\033[1m[{i}] {choices[i]}\033[0m\n"
+                    f"    \033[3;34m{lbl}\033[0m"
+                )
+            choice = input("> ")
+
+            image_tag = choices[int(choice)]
+
+        image = images.get(image_tag)
+        print("Selected image:")
+        print("---------------------------")
+        print(yaml.dump(image, default_flow_style=True))
+        print("---------------------------")
+
+        image_setup = setup.get(image.get('setup'))
+
+        env = image_setup.get('env')
+        args = {
+            # General
+            "ARCH": image.get('arch'),
+            "BASE": image.get('base'),
+            "SETUP_ID": image.get('setup'),
+            # Environment
+            "CC": env.get('CC', ''),
+            "CXX": env.get('CXX', ''),
+            "FC": env.get('FC', ''),
+            "GCOV": env.get('GCOV', ''),
+            "MPICH_CC": env.get('MPICH_CC', ''),
+            "MPICH_CXX": env.get('MPICH_CXX', '')
+        }
+
+        space = ' '
+        cmd = ("docker build . "
+                f" --tag {image_tag}"
+                f" --file {os.path.dirname(__file__)}/docker/base.dockerfile"
+                f" {space.join([f'--build-arg {k}={v}' for (k,v) in args.items()])}"
+                " --progress=plain"
+                " --no-cache"
+            )
+        print(cmd)
+        os.system(cmd)
+
+        # TODO: option to push to Dockerhub
+
+DockerBuilder().build(sys.argv[1:])

ci/build-matrix.py

@@ -0,0 +1,71 @@
+import copy
+import os
+import json
+
+from util import resolve_conf
+import yaml
+
+
+class MatrixGenerator:
+    """MatrixGenerator to generate a matrix file for Github and Azure Pipelines"""
+
+    def generate(self):
+        """Generate a matrix of runners and inner environments to be used by CI pipelines"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+        config = resolve_conf(copy.deepcopy(raw_config))
+
+        for runner_type in ["github", "azure-pipelines"]:
+            runners = [runner for runner in config.get("runners")
+                if runner.get("type") == runner_type]
+
+            matrix = []
+            for runner in runners:
+                matrix_item = {
+                    "label": runner.get("label"),
+                    "runs-on": runner.get("runs-on")
+                }
+
+                if runner.get("setup") is not None:
+                    setup = config.get("setup").get(runner.get("setup"))
+
+                    if setup is None:
+                        raise RuntimeError(f"Setup not found {runner.get('setup')}")
+
+                    matrix_item["setup"] = runner.get("setup")
+
+                if runner.get("image") is not None:
+                    image_name = (runner.get("image", {}).get("repository", "") + ":"
+                                + runner.get("image", {}).get("tag", ""))
+                    image = config.get("images").get(image_name)
+
+                    if image is None:
+                        raise RuntimeError(f"Image not found {runner.get('image')}")
+
+                    setup = config.get("setup").get(image.get("setup"))
+                    if setup is None:
+                        raise RuntimeError(f"Setup not found {runner.get('setup')}")
+
+                    matrix_item["image"] = image.get("repository") + ":" + image.get("tag")
+
+                    if matrix_item["label"] is None:
+                        matrix_item["label"] = image.get("label")
+
+                if matrix_item["label"] is None:
+                    matrix_item["label"] = setup.get("label")
+
+                matrix.append(matrix_item)
+
+            data = json.dumps({
+                "_comment": "This file has been generated. Please do not edit",
+                "matrix": matrix}, indent=2)
+            with open(
+                os.path.dirname(__file__) + f"/shared/matrix/{runner_type}.json",
+                'w+',
+                encoding="utf-8"
+            ) as file:
+                file.write(data)
+
+MatrixGenerator().generate()

ci/build-setup.py

@@ -0,0 +1,64 @@
+import copy
+import os
+from typing import List
+
+from util import resolve_conf
+import yaml
+
+class SetupBuilder:
+    """Dockerfile generator class"""
+
+    def __instructions(self, dep_id, args: list) -> List[str]:
+        """ Generate shell instructions to setup a dependency"""
+        call_args = []
+        # repeat instructions if args is an array of array
+        if args is not None and len(args) > 0:
+            if isinstance(args[0], list):
+                instructions = []
+                for (_, sub_args) in enumerate(args):
+                    instructions.extend(self.__instructions(dep_id, sub_args))
+                return instructions
+
+            call_args = [ f"\"{a}\"" for a in args]
+
+        cmd = f"./{dep_id}.sh"
+        if len(call_args) > 0:
+            cmd = f"{cmd} {' '.join(call_args)}"
+        return [ cmd ]
+
+    def build(self):
+        """Build setup scripts for each setup configuration defined in config"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+        config = resolve_conf(copy.deepcopy(raw_config))
+
+        setup = config.get("setup")
+        for (setup_id, setup_config) in setup.items():
+            # generate install instructions and install dependencies commands
+            instructions = []
+            downloads = []
+            for (dep_id, args) in setup_config.get("deps").items():
+                downloads.append(f"wget $SCRIPTS_DEPS_URL/{dep_id}.sh")
+                instructions.extend(self.__instructions(dep_id, args))
+
+            setup_script = ""
+            with open(
+                os.path.dirname(__file__) + '/setup-template.sh',
+                'r',
+                encoding="utf-8"
+            ) as file:
+                setup_script = file.read()
+            setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', setup_config.get("label"))
+            setup_script = setup_script.replace('%DEPS_DOWNLOAD%', '\n'.join(downloads))
+            setup_script = setup_script.replace('%DEPS_INSTALL%', '\n'.join(instructions))
+
+            setup_filename = f"setup-{setup_id}.sh"
+            setup_filepath = os.path.join(os.path.dirname(__file__),
+                                          'shared', 'scripts', setup_filename)
+
+            with open(setup_filepath, "w+", encoding="utf-8") as f:
+                f.write(setup_script)
+
+SetupBuilder().build()