From e29c6cee7a570303d0d60c5743a0327bbbcbdafc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Dutheillet-Lamonth=C3=A9zie?= Date: Fri, 27 Sep 2024 18:33:53 +0200 Subject: [PATCH] #2: rename ci python scripts --- .github/workflows/build-docker-images.yml | 2 +- ci/build-docker-image.py | 85 +++++++++++++++++++++++ ci/build-matrix.py | 71 +++++++++++++++++++ ci/build-setup.py | 64 +++++++++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100755 ci/build-docker-image.py create mode 100755 ci/build-matrix.py create mode 100755 ci/build-setup.py diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index bb4e7d81..cda76995 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -42,4 +42,4 @@ jobs: - run: pip install pyyaml - name: Docker image build run: | - python ci/build-docker-image ${{ matrix.runner.image }} + python ci/build-docker-image.py ${{ matrix.runner.image }} diff --git a/ci/build-docker-image.py b/ci/build-docker-image.py new file mode 100755 index 00000000..76778334 --- /dev/null +++ b/ci/build-docker-image.py @@ -0,0 +1,85 @@ +import copy +import os +import sys + +from util import resolve_conf +import yaml + +class DockerBuilder: + """Dockerfile generator class""" + + def build(self, args: list): + """Build an image using a given docker configuration fro the config file""" + + raw_config: dict = {} + with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file: + raw_config = yaml.safe_load(file) + + config = resolve_conf(copy.deepcopy(raw_config)) + images = config.get("images") + setup = config.get("setup") + + image_tag = None + if len(args) > 0: + image_tag = args[0] + if image_tag not in images.keys(): + print(f"[error] Image not found {image_tag}.\n" + f"Available images:{(os.linesep + '- ')}" + f"{(os.linesep + '- ') . join(images.keys())}") + raise SystemExit(1) + else: + # Step 1: list platforms and their configurations + choices = {k: v for k, v in enumerate(images.keys())} + print("Choose image: ") + for i in choices: + image = images.get(choices[i]) + setup_id = image.get('setup') + current_setup = setup.get(setup_id) + if current_setup is None: + raise RuntimeError(f"Invalid setup {setup_id}") + lbl = current_setup.get('label', image.get('setup')) + print( + f"\033[1m[{i}] {choices[i]}\033[0m\n" + f" \033[3;34m{lbl}\033[0m" + ) + choice = input("> ") + + image_tag = choices[int(choice)] + + image = images.get(image_tag) + print("Selected image:") + print("---------------------------") + print(yaml.dump(image, default_flow_style=True)) + print("---------------------------") + + image_setup = setup.get(image.get('setup')) + + env = image_setup.get('env') + args = { + # General + "ARCH": image.get('arch'), + "BASE": image.get('base'), + "SETUP_ID": image.get('setup'), + # Environment + "CC": env.get('CC', ''), + "CXX": env.get('CXX', ''), + "FC": env.get('FC', ''), + "GCOV": env.get('GCOV', ''), + "MPICH_CC": env.get('MPICH_CC', ''), + "MPICH_CXX": env.get('MPICH_CXX', '') + } + + space = ' ' + cmd = ("docker build . " + f" --tag {image_tag}" + f" --file {os.path.dirname(__file__)}/docker/base.dockerfile" + f" {space.join([f'--build-arg {k}={v}' for (k,v) in args.items()])}" + " --progress=plain" + " --no-cache" + ) + print(cmd) + os.system(cmd) + + # TODO: option to push to Dockerhub + +DockerBuilder().build(sys.argv[1:]) diff --git a/ci/build-matrix.py b/ci/build-matrix.py new file mode 100755 index 00000000..1faa70b4 --- /dev/null +++ b/ci/build-matrix.py @@ -0,0 +1,71 @@ +import copy +import os +import json + +from util import resolve_conf +import yaml + + +class MatrixGenerator: + """MatrixGenerator to generate a matrix file for Github and Azure Pipelines""" + + def generate(self): + """Generate a matrix of runners and inner environments to be used by CI pipelines""" + + raw_config: dict = {} + with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file: + raw_config = yaml.safe_load(file) + config = resolve_conf(copy.deepcopy(raw_config)) + + for runner_type in ["github", "azure-pipelines"]: + runners = [runner for runner in config.get("runners") + if runner.get("type") == runner_type] + + matrix = [] + for runner in runners: + matrix_item = { + "label": runner.get("label"), + "runs-on": runner.get("runs-on") + } + + if runner.get("setup") is not None: + setup = config.get("setup").get(runner.get("setup")) + + if setup is None: + raise RuntimeError(f"Setup not found {runner.get('setup')}") + + matrix_item["setup"] = runner.get("setup") + + if runner.get("image") is not None: + image_name = (runner.get("image", {}).get("repository", "") + ":" + + runner.get("image", {}).get("tag", "")) + image = config.get("images").get(image_name) + + if image is None: + raise RuntimeError(f"Image not found {runner.get('image')}") + + setup = config.get("setup").get(image.get("setup")) + if setup is None: + raise RuntimeError(f"Setup not found {runner.get('setup')}") + + matrix_item["image"] = image.get("repository") + ":" + image.get("tag") + + if matrix_item["label"] is None: + matrix_item["label"] = image.get("label") + + if matrix_item["label"] is None: + matrix_item["label"] = setup.get("label") + + matrix.append(matrix_item) + + data = json.dumps({ + "_comment": "This file has been generated. Please do not edit", + "matrix": matrix}, indent=2) + with open( + os.path.dirname(__file__) + f"/shared/matrix/{runner_type}.json", + 'w+', + encoding="utf-8" + ) as file: + file.write(data) + +MatrixGenerator().generate() diff --git a/ci/build-setup.py b/ci/build-setup.py new file mode 100755 index 00000000..fa21e8cb --- /dev/null +++ b/ci/build-setup.py @@ -0,0 +1,64 @@ +import copy +import os +from typing import List + +from util import resolve_conf +import yaml + +class SetupBuilder: + """Dockerfile generator class""" + + def __instructions(self, dep_id, args: list) -> List[str]: + """ Generate shell instructions to setup a dependency""" + call_args = [] + # repeat instructions if args is an array of array + if args is not None and len(args) > 0: + if isinstance(args[0], list): + instructions = [] + for (_, sub_args) in enumerate(args): + instructions.extend(self.__instructions(dep_id, sub_args)) + return instructions + + call_args = [ f"\"{a}\"" for a in args] + + cmd = f"./{dep_id}.sh" + if len(call_args) > 0: + cmd = f"{cmd} {' '.join(call_args)}" + return [ cmd ] + + def build(self): + """Build setup scripts for each setup configuration defined in config""" + + raw_config: dict = {} + with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file: + raw_config = yaml.safe_load(file) + config = resolve_conf(copy.deepcopy(raw_config)) + + setup = config.get("setup") + for (setup_id, setup_config) in setup.items(): + # generate install instructions and install dependencies commands + instructions = [] + downloads = [] + for (dep_id, args) in setup_config.get("deps").items(): + downloads.append(f"wget $SCRIPTS_DEPS_URL/{dep_id}.sh") + instructions.extend(self.__instructions(dep_id, args)) + + setup_script = "" + with open( + os.path.dirname(__file__) + '/setup-template.sh', + 'r', + encoding="utf-8" + ) as file: + setup_script = file.read() + setup_script = setup_script.replace('%ENVIRONMENT_LABEL%', setup_config.get("label")) + setup_script = setup_script.replace('%DEPS_DOWNLOAD%', '\n'.join(downloads)) + setup_script = setup_script.replace('%DEPS_INSTALL%', '\n'.join(instructions)) + + setup_filename = f"setup-{setup_id}.sh" + setup_filepath = os.path.join(os.path.dirname(__file__), + 'shared', 'scripts', setup_filename) + + with open(setup_filepath, "w+", encoding="utf-8") as f: + f.write(setup_script) + +SetupBuilder().build()