Merge branch 'develop' into blue/build-image-workflow

.gitignore

@@ -9,3 +9,5 @@ poetry.lock
 *.gvmi.descr.bin
 .requirements.txt
 .python-version
+golem-cluster.dev.yaml
+golem-cluster.local.yaml

golem-cluster.override.1-source-files.yaml

@@ -0,0 +1,16 @@
+# The files or directories to copy to the head and worker nodes
+file_mounts:
+  # remote_path: local_path
+  {
+    "/app/ray_on_golem": "./ray_on_golem",
+    "/app/golem": "../golem-core-python/golem",
+  }
+
+rsync_exclude: [
+  "**/__pycache__",
+]
+
+# List of commands that will be run to initialize the nodes (before `setup_commands`)
+initialization_commands: [
+  "cp -fR /app/golem/* $(python -c 'import site; print(site.getsitepackages()[0])')/golem"
+]

golem-cluster.override.2-image.yaml

@@ -0,0 +1,5 @@
+provider:
+  parameters:
+    node_config:
+      demand:
+        image_tag: "blueshade/ray-on-golem:0.2.1-py3.10.13-ray2.7.1"

golem-cluster.override.3-disable-stats.yaml

@@ -0,0 +1,2 @@
+provider:
+    enable_registry_stats: false

golem-cluster.yaml

@@ -21,11 +21,11 @@ provider:
     # Polygon is for mainnet operations.
     network: "goerli"
 
-    # Maximum amount of GML that's going to be spent (not supported yet)
+    # Maximum amount of GLMs that's going to be spent for the whole cluster
     budget: 2
 
-    # Params for creating golem demands (same for head and workers)
     node_config:
+      # Parameters for golem demands (same for head and workers)
       demand:
         # if not provided, image_tag will be autodetected based on currently used python and ray versions
         # check available versions at https://registry.golem.network/explore/golem/ray-on-golem

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ray-on-golem"
-version = "0.2.1"
+version = "0.3.0-dev"
 description = "Golem Network integration with Ray"
 authors = ["Golem Factory <[email protected]>"]
 readme = "README.md"
@@ -44,6 +44,7 @@ black = "^23.7.0"
 isort = "^5.12.0"
 autoflake = "^2.2.0"
 gvmkit-build = "^0.3.13"
+dpath = "^2.1.6"
 yamlpath = "^3.8.1"
 
 [build-system]
@@ -58,6 +59,7 @@ _format_black = "black ."
 check_license = {sequence = ["_check_license_export", "_check_license_verify"], help = "Check license compatibility"}
 _check_license_export = "poetry export -f requirements.txt -o .requirements.txt"
 _check_license_verify = "liccheck -r .requirements.txt"
+dev_yaml = {cmd = "python -m utils.apply_overrides -o golem-cluster.dev.yaml golem-cluster.override.*", help="Generate development YAML file."}
 
 [tool.isort]
 profile = "black"

utils/apply_overrides.py

@@ -0,0 +1,45 @@
+import argparse
+from pathlib import Path
+from typing import Final
+import sys
+import yaml
+
+from .yaml import load_yamls
+
+BASE_YAML: Final = Path("golem-cluster.yaml")
+LOCAL_OVERRIDE_YAML: Final = Path("golem-cluster.local.yaml")
+
+
+def main():
+    parser = argparse.ArgumentParser("Apply YAML overrides and output a complete YAML.")
+    parser.add_argument("overrides", type=Path, nargs="*", help="Overrides to apply")
+    parser.add_argument(
+        "--base", "-b", type=Path, default=BASE_YAML, help="Base YAML file, default: %(default)s"
+    )
+    parser.add_argument(
+        "--local",
+        "-l",
+        type=Path,
+        default=LOCAL_OVERRIDE_YAML,
+        help="Local override file, default: %(default)s",
+    )
+    parser.add_argument("--out", "-o", type=Path, help="Output file, default: stdout")
+    args = parser.parse_args()
+
+    yaml_files = [args.base]
+    yaml_files.extend(args.overrides)
+
+    if args.local.exists():
+        yaml_files.append(args.local)
+
+    data = load_yamls(*yaml_files)
+
+    if args.out:
+        with open(args.out, "w") as f:
+            yaml.dump(data, f)
+    else:
+        yaml.dump(data, sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

utils/yaml.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+from typing import Any, Dict
+
+import dpath.util
+import yaml
+
+
+def load_yamls(*yaml_paths: Path) -> Dict[str, Any]:
+    """Load the provided YAML files, merging their contents in a deep manner.
+
+    The order of the files is relevant, that is: the first YAML is considered the base.
+    All the remaining files are loaded one by one and deeply merged into the base.
+
+    Returns a dict representing the result of all YAML files merged into the first one.
+    """
+
+    def _load_yaml(path: Path) -> Dict[str, Any]:
+        with path.open() as f:
+            return yaml.load(f, yaml.SafeLoader)
+
+    base_dict = _load_yaml(yaml_paths[0])
+    for path in yaml_paths[1:]:
+        data = _load_yaml(path)
+        dpath.util.merge(
+            base_dict,
+            data,
+        )
+
+    return base_dict