diff --git a/.gitignore b/.gitignore index ff02e19..e59d491 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,9 @@ venv.bak/ # JetBrains IDEs .idea/ +# Vscode +.vscode + # Rope project settings .ropeproject diff --git a/tests/test_yaml/test1.expected.valohai.yaml b/tests/test_yaml/test1.expected.valohai.yaml index a794c27..ab09bdb 100644 --- a/tests/test_yaml/test1.expected.valohai.yaml +++ b/tests/test_yaml/test1.expected.valohai.yaml @@ -28,12 +28,12 @@ type: float inputs: - name: input1 - default: asdf + default: + - datum://asdf optional: false - name: input2 default: - - yolol - - yalala + - datum://yolol optional: false - name: my-optional-input optional: true diff --git a/tests/test_yaml/test1.py b/tests/test_yaml/test1.py index f0f1a7d..8aaff55 100644 --- a/tests/test_yaml/test1.py +++ b/tests/test_yaml/test1.py @@ -7,7 +7,14 @@ "param4": 0.0001, } -inputs = {"input1": "asdf", "input2": ["yolol", "yalala"], "my-optional-input": ""} +inputs = { + "input1": "datum://asdf", + "input2": [ + "datum://yolol", + "yalala", # this local path should be ignored + ], + "my-optional-input": "", +} def prepare(a, b): diff --git a/tests/test_yaml/test2.expected.valohai.yaml b/tests/test_yaml/test2.expected.valohai.yaml index 92bf689..a7c30fe 100644 --- a/tests/test_yaml/test2.expected.valohai.yaml +++ b/tests/test_yaml/test2.expected.valohai.yaml @@ -22,10 +22,11 @@ type: float inputs: - name: input1 - default: asdf + default: + - datum://asdf optional: false - name: input2 default: - - yolol - - yalala + - datum://yolol + - datum://yalala optional: false diff --git a/tests/test_yaml/test2.py b/tests/test_yaml/test2.py index b47d41e..db43172 100644 --- a/tests/test_yaml/test2.py +++ b/tests/test_yaml/test2.py @@ -7,7 +7,7 @@ "param4": 0.0001, } -inputs = {"input1": "asdf", "input2": ["yolol", "yalala"]} +inputs = {"input1": "datum://asdf", "input2": ["datum://yolol", "datum://yalala"]} def prepare(a, b): diff --git a/tests/test_yaml/test3.expected.valohai.yaml b/tests/test_yaml/test3.expected.valohai.yaml index 0ef10aa..34139d8 100644 --- a/tests/test_yaml/test3.expected.valohai.yaml +++ b/tests/test_yaml/test3.expected.valohai.yaml @@ -24,10 +24,9 @@ type: float inputs: - name: input1 - default: asdf - optional: false + optional: true - name: input2 default: - - yolol - - yalala + - datum://yolol + - datum://yalala optional: false diff --git a/tests/test_yaml/test3.py b/tests/test_yaml/test3.py index b47d41e..6bd84b3 100644 --- a/tests/test_yaml/test3.py +++ b/tests/test_yaml/test3.py @@ -7,7 +7,10 @@ "param4": 0.0001, } -inputs = {"input1": "asdf", "input2": ["yolol", "yalala"]} +inputs = { + "input1": "asdf", # local path + "input2": ["datum://yolol", "datum://yalala"], +} def prepare(a, b): diff --git a/tests/test_yaml/test4.expected.valohai.yaml b/tests/test_yaml/test4.expected.valohai.yaml index 9a6ebd0..2b51013 100644 --- a/tests/test_yaml/test4.expected.valohai.yaml +++ b/tests/test_yaml/test4.expected.valohai.yaml @@ -22,14 +22,9 @@ type: float inputs: - name: input1 - default: asdf/* - keep-directories: suffix - optional: false + optional: true - name: input2 - default: - - yolol - - yalala - optional: false + optional: true - step: name: herpderp image: busybox diff --git a/tests/test_yaml/test4.py b/tests/test_yaml/test4.py index 9f838b9..dc15f50 100644 --- a/tests/test_yaml/test4.py +++ b/tests/test_yaml/test4.py @@ -7,7 +7,10 @@ "param4": 0.0001, } -inputs = {"input1": "asdf/*", "input2": ["yolol", "yalala"]} +inputs = { + "input1": "asdf/*", + "input2": ["yolol", "yalala"], +} def prepare(a, b): diff --git a/tests/test_yaml/test7.expected.valohai.yaml b/tests/test_yaml/test7.expected.valohai.yaml index ca9e8cf..4187d26 100644 --- a/tests/test_yaml/test7.expected.valohai.yaml +++ b/tests/test_yaml/test7.expected.valohai.yaml @@ -25,7 +25,8 @@ type: string inputs: - name: my-image - default: https://dist.valohai.com/valohai-utils-tests/Example.jpg + default: + - https://dist.valohai.com/valohai-utils-tests/Example.jpg optional: false - name: my-optional-input optional: true diff --git a/tests/test_yaml/test8.expected.valohai.yaml b/tests/test_yaml/test8.expected.valohai.yaml index 647a049..80a3cd8 100644 --- a/tests/test_yaml/test8.expected.valohai.yaml +++ b/tests/test_yaml/test8.expected.valohai.yaml @@ -35,5 +35,6 @@ default: s3://special-bucket/images/**.jpg optional: false - name: weights - default: s3://special-bucket/weights/yolo.pb + default: + - s3://special-bucket/weights/yolo.pb optional: false diff --git a/valohai/internals/utils.py b/valohai/internals/utils.py index 2418b07..0242a1a 100644 --- a/valohai/internals/utils.py +++ b/valohai/internals/utils.py @@ -18,3 +18,9 @@ def get_sha256_hash(filepath: str) -> str: with open(filepath, "rb") as f: return get_fp_sha256(f) # type: ignore + + +def is_local_file_path(path_str: str) -> bool: + if "://" in path_str: + return False + return True diff --git a/valohai/internals/yaml.py b/valohai/internals/yaml.py index d81cc74..0fe530d 100644 --- a/valohai/internals/yaml.py +++ b/valohai/internals/yaml.py @@ -3,6 +3,7 @@ from valohai_yaml.objs import Config, Parameter, Step from valohai_yaml.objs.input import Input, KeepDirectories +from valohai_yaml.utils import listify from valohai.consts import DEFAULT_DOCKER_IMAGE from valohai.internals.notebooks import ( @@ -12,6 +13,7 @@ ) from valohai.internals.parsing import parse from valohai.types import InputDict, ParameterDict +from valohai.internals.utils import is_local_file_path def generate_step( @@ -27,7 +29,6 @@ def generate_step( # We need to generate a POSIX-compliant command, even if we are running this method in Windows # The path separator must be forced to POSIX relative_source_path = relative_source_path.replace(os.sep, "/") - config_step = Step( name=step, image=image, @@ -41,7 +42,6 @@ def generate_step( for key, value in inputs.items(): config_step.inputs[key] = parse_input(key, value) - return config_step @@ -70,20 +70,16 @@ def parse_input(key: str, value: Any) -> Input: value["name"] = key return Input.parse(value) - has_wildcards = any( - "*" in uri for uri in ([value] if isinstance(value, str) else value) - ) - empty_default = ( - not value - or isinstance(value, List) - and (len(value) == 0 or len(value) == 1 and not value[0]) - ) + # Check and ignore the local path file + checked_value = [val for val in listify(value) if not is_local_file_path(val)] + + has_wildcards = any("*" in uri for uri in checked_value) keep_directories = KeepDirectories.SUFFIX if has_wildcards else KeepDirectories.NONE return Input( name=key, - default=None if empty_default else value, - optional=empty_default, + default=checked_value if checked_value else None, + optional=not checked_value, keep_directories=keep_directories, )