Merge branch 'main' of github.com:latchbio/latch into ayush/cp-conn-r…

…euse

CHANGELOG.md

@@ -16,6 +16,33 @@ Types of changes
 
 # Latch SDK Changelog
 
+## v2.53.0 - 2024-09-28
+
+### Changed
+
+* Make OFS the default storage option for Nextflow workflows
+* Bump the `nextflow` base image to `v2.1.2`
+
+## 2.52.5 - 2024-09-23
+
+### Changed
+
+* Improve error messaging when registering Nextflow workflows
+
+## 2.52.4 - 2024-09-19
+
+### Changed
+
+* Nextflow
+  - Fix typo in Nextflow base image
+
+## 2.52.3 - 2024-09-17
+
+### Changed
+
+* Nextflow
+  - Update `nextflow` base image to version `v2.0.0`
+
 ## 2.52.2 - 2024-09-04
 
 ### Changed

latch_cli/constants.py

@@ -22,7 +22,7 @@ class LatchConstants:
     base_image: str = (
         "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main"
     )
-    nextflow_latest_version: str = "v1.1.8"
+    nextflow_latest_version: str = "v2.1.2"
 
     file_max_size: int = 4 * Units.MiB
     file_chunk_size: int = 64 * Units.MiB

latch_cli/nextflow/config.py

@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 from textwrap import dedent
-from typing import Any, Optional, Type
+from typing import Optional, Type, Union
 
 import click
 
@@ -11,10 +11,15 @@
 from latch_cli.snakemake.utils import reindent
 
 
-def get_param_type(details: dict) -> Type:
+def get_param_type(
+    details: dict,
+) -> Union[
+    Type[str], Type[bool], Type[int], Type[float], Type["LatchFile"], Type["LatchDir"]
+]:
     t = details.get("type")
     if t is None:
-        return Any
+        # rahul: assume string if type is not specified
+        return str
 
     if t == "string":
         format = details.get("format")
@@ -31,7 +36,7 @@ def get_param_type(details: dict) -> Type:
     elif t == "number":
         return float
 
-    return Any
+    return str
 
 
 def generate_metadata(
@@ -108,6 +113,11 @@ def generate_metadata(
             default = None
             if generate_defaults and t not in {LatchFile, LatchDir, LatchOutputDir}:
                 default = details.get("default")
+                if default is not None:
+                    try:
+                        default = t(default)
+                    except ValueError:
+                        pass
 
             if param not in required_params:
                 t = Optional[t]

latch_cli/nextflow/workflow.py

@@ -1,3 +1,4 @@
+import json
 from dataclasses import fields, is_dataclass
 from enum import Enum
 from pathlib import Path
@@ -10,9 +11,11 @@
 from latch.types.directory import LatchDir, LatchOutputDir
 from latch.types.file import LatchFile
 from latch.types.metadata import NextflowParameter
+from latch_cli.constants import latch_constants
 from latch_cli.snakemake.config.utils import get_preamble, type_repr
 from latch_cli.snakemake.utils import reindent
 from latch_cli.utils import identifier_from_str, urljoins
+from latch_cli.workflow_config import LatchWorkflowConfig
 
 template = """\
 import sys
@@ -54,10 +57,11 @@ def initialize() -> str:
 
     print("Provisioning shared storage volume... ", end="")
     resp = requests.post(
-        "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage",
+        "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage-ofs",
         headers=headers,
         json={{
             "storage_expiration_hours": {storage_expiration_hours},
+            "version": {nextflow_version},
         }},
     )
     resp.raise_for_status()
@@ -268,6 +272,29 @@ def get_results_code_block(parameters: Dict[str, NextflowParameter]) -> str:
     return code_block
 
 
+def get_nextflow_major_version(pkg_root: Path) -> int:
+    try:
+        with (pkg_root / latch_constants.pkg_config).open("r") as f:
+            config = LatchWorkflowConfig(**json.load(f))
+    except FileNotFoundError:
+        click.secho(
+            dedent(f"""
+            Could not find the latch config file at {pkg_root / latch_constants.pkg_config}
+
+            Please check if you package root contains a Dockerfile that was NOT generated
+            by the Latch CLI. If it does, please move it to a subdirectory and try again.
+            """),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    if "latch-base-nextflow" not in config.base_image:
+        return 1
+
+    version = config.base_image.split(":")[-1]
+    return int(version[1])
+
+
 def generate_nextflow_workflow(
     pkg_root: Path,
     metadata_root: Path,
@@ -415,6 +442,7 @@ def generate_nextflow_workflow(
         storage_gib=resources.storage_gib,
         storage_expiration_hours=resources.storage_expiration_hours,
         log_dir=log_dir,
+        nextflow_version=get_nextflow_major_version(pkg_root),
     )
 
     dest.write_text(entrypoint)

latch_cli/services/register/register.py

@@ -6,6 +6,7 @@
 import time
 import webbrowser
 from pathlib import Path
+from textwrap import dedent
 from typing import Iterable, List, Optional
 
 import click
@@ -228,6 +229,29 @@ def _build_and_serialize(
         exit_status = ctx.dkr_client.wait(container_id)
         if exit_status["StatusCode"] != 0:
             click.secho("\nWorkflow failed to serialize", fg="red", bold=True)
+            if "TypeTransformerFailedError" in "".join(serialize_logs):
+                file_to_update = (
+                    "latch_metadata/parameters.py"
+                    if ctx.workflow_type
+                    in (WorkflowType.nextflow, WorkflowType.snakemake)
+                    else "workflow function"
+                )
+                click.secho(
+                    dedent(f"""
+                    This is likely caused by a type mismatch between the type of one of
+                    your input parameters and the type of the default value for that
+                    parameter.
+
+                    To fix this, ensure that the default value for each parameter matches
+                    the type of the parameter itself in your {file_to_update}.
+
+                    For example, if you have a parameter `x: float` and the default value
+                    `x = 200000`, you will need to update the default value of x to
+                    `200000.0`.
+                    """),
+                    fg="red",
+                )
+
             sys.exit(1)
 
         ctx.dkr_client.remove_container(container_id)

setup.py

@@ -13,7 +13,7 @@
 
 setup(
     name="latch",
-    version="v2.52.2",
+    version="v2.53.0",
     author_email="[email protected]",
     description="The Latch SDK",
     packages=find_packages(),