From ed85d9907d5fbbe80f419bb0278da8c7f8747a74 Mon Sep 17 00:00:00 2001 From: Omar Tarabai Date: Wed, 9 Oct 2024 13:36:03 +0300 Subject: [PATCH] Add PERIAN Job Platform example Signed-off-by: Omar Tarabai --- _example_template/Dockerfile | 2 +- docs/integrations/index.md | 3 ++ examples/perian_agent/Dockerfile | 31 ++++++++++++++ examples/perian_agent/README.md | 21 ++++++++++ .../perian_agent/perian_agent/__init__.py | 0 examples/perian_agent/perian_agent/example.py | 40 +++++++++++++++++++ examples/perian_agent/requirements.in | 5 +++ 7 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 examples/perian_agent/Dockerfile create mode 100644 examples/perian_agent/README.md create mode 100644 examples/perian_agent/perian_agent/__init__.py create mode 100644 examples/perian_agent/perian_agent/example.py create mode 100644 examples/perian_agent/requirements.in diff --git a/_example_template/Dockerfile b/_example_template/Dockerfile index 65449f624..d56c81ef4 100644 --- a/_example_template/Dockerfile +++ b/_example_template/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.8-slim-buster -LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks +LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks WORKDIR /root ENV VENV /opt/venv diff --git a/docs/integrations/index.md b/docs/integrations/index.md index 283ac34d7..b5ef32611 100644 --- a/docs/integrations/index.md +++ b/docs/integrations/index.md @@ -108,6 +108,8 @@ Native backend plugins can be executed without any external service dependencies - Execute tasks using the MemVerge Memory Machine Cloud agent. * - {doc}`OpenAI Batch ` - Submit requests for asynchronous batch processing on OpenAI. +* - {doc}`PERIAN Job Platform agent ` + - Execute tasks on PERIAN Job Platform. * - {doc}`Sensor agent ` - Run sensor jobs in your workflows with the sensor agent. * - {doc}`Snowflake agent ` @@ -235,6 +237,7 @@ ChatGPT agent Databricks agent Memory Machine Cloud agent OpenAI batch agent +PERIAN Job Platform agent Sensor agent Snowflake agent ``` diff --git a/examples/perian_agent/Dockerfile b/examples/perian_agent/Dockerfile new file mode 100644 index 000000000..d56c81ef4 --- /dev/null +++ b/examples/perian_agent/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.8-slim-buster +LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV VENV /opt/venv +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONPATH /root + +# This is necessary for opencv to work +RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential curl + +WORKDIR /root + +ENV VENV /opt/venv +# Virtual environment +RUN python3 -m venv ${VENV} +ENV PATH="${VENV}/bin:$PATH" + +# Install Python dependencies +COPY requirements.in /root +RUN pip install -r /root/requirements.in +RUN pip freeze + +# Copy the actual code +COPY . /root + +# This tag is supplied by the build script and will be used to determine the version +# when registering tasks, workflows, and launch plans +ARG tag +ENV FLYTE_INTERNAL_IMAGE $tag diff --git a/examples/perian_agent/README.md b/examples/perian_agent/README.md new file mode 100644 index 000000000..a07769d9d --- /dev/null +++ b/examples/perian_agent/README.md @@ -0,0 +1,21 @@ +```{eval-rst} +.. tags:: Cloud, GPU, Integration, Advanced +``` + +(perian_agent)= + +# PERIAN Job Platform Agent + +The PERIAN Flyte Agent enables you to execute Flyte tasks on the [PERIAN Sky Platform](https://perian.io/). PERIAN allows the execution of any task on servers aggregated from multiple cloud providers. + +Example usage: + +```{auto-examples-toc} +example +``` + +To get started with PERIAN, see the [PERIAN documentation](https://perian.io/docs/overview) and the [PERIAN Flyte Agent documentation](https://perian.io/docs/flyte-getting-started). + +## Agent setup + +Consult the [PERIAN Flyte Agent setup guide](https://perian.io/docs/flyte-setup-guide). diff --git a/examples/perian_agent/perian_agent/__init__.py b/examples/perian_agent/perian_agent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/perian_agent/perian_agent/example.py b/examples/perian_agent/perian_agent/example.py new file mode 100644 index 000000000..a23060be2 --- /dev/null +++ b/examples/perian_agent/perian_agent/example.py @@ -0,0 +1,40 @@ +# %% [markdown] +# (example)= +# # PERIAN agent example usage +# +# This example shows how to use the PERIAN agent to execute tasks on PERIAN Job Platform. + +# %% +from flytekit import ImageSpec, task, workflow +from flytekitplugins.perian_job import PerianConfig + +image_spec = ImageSpec( + name="flyte-test", + registry="my-registry", + python_version="3.11", + apt_packages=["wget", "curl", "git"], + packages=[ + "flytekitplugins-perian-job", + ], +) + + +# %% [markdown] +# `PerianConfig` configures `PerianTask`. Tasks specified with `PerianConfig` will be executed on PERIAN Job Platform. + + +# %% +@task( + container_image=image_spec, + task_config=PerianConfig( + accelerators=1, + accelerator_type="A100", + ), +) +def perian_hello(name: str) -> str: + return f"hello {name}!" + + +@workflow +def my_wf(name: str = "world") -> str: + return perian_hello(name=name) diff --git a/examples/perian_agent/requirements.in b/examples/perian_agent/requirements.in new file mode 100644 index 000000000..b0b2423b0 --- /dev/null +++ b/examples/perian_agent/requirements.in @@ -0,0 +1,5 @@ +flytekit>=1.7.0 +wheel +matplotlib +flytekitplugins-deck-standard +flytekitplugins-perian-job