diff --git a/README.md b/README.md index bca345f..c4203be 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ with DAG('navn-dag', start_date=days_ago(1), schedule_interval="*/10 * * * *") a slack_channel="<#slack-alarm-kanal>") ``` -## Kubernetes operator +### Kubernetes operator Vi tilbyr også vår egen Kubernetes operator som kloner et valg repo inn i containeren. @@ -85,3 +85,33 @@ with DAG('navn-dag', start_date=days_ago(1), schedule_interval="*/10 * * * *") a image="europe-north1-docker.pkg.dev/nais-management-233d/ditt-team/ditt-image:din-tag", slack_channel="<#slack-alarm-kanal>") ``` + +## Sette resource requirements + +Vi har støtte for å sette `requests` og `limits` for hver operator. +Merk at man ikke trenger å sette `limits` på CPU da dette blir automatisk løst av plattformen. + +Ved å bruke `ephemeral-storage` kan man be om ekstra diskplass for lagring i en task. + +```python +from airflow import DAG +from airflow.utils.dates import days_ago +from dataverk_airflow import python_operator + + +with DAG('navn-dag', start_date=days_ago(1), schedule_interval="*/10 * * * *") as dag: + t1 = python_operator(dag=dag, + name="", + repo="navikt/", + script_path="/path/to/script.py", + resources={ + "requests": { + "memory": "50Mi", + "cpu": "100m", + "ephemeral-storage": "1Gi" + }, + "limits": { + "memory": "100Mi" + } + }) +``` diff --git a/dataverk_airflow/kubernetes_operator.py b/dataverk_airflow/kubernetes_operator.py index b86f5e5..007cf0c 100644 --- a/dataverk_airflow/kubernetes_operator.py +++ b/dataverk_airflow/kubernetes_operator.py @@ -64,7 +64,7 @@ def kubernetes_operator( :param extra_envs: dict: dict with environment variables example: {"key": "value", "key2": "value2"} :param allowlist: list: list of hosts and port the task needs to reach on the format host:port :param requirements_path: bool: Path (including filename) to your requirements.txt - :param resources: dict: Specify required cpu and memory requirements (keys in dict: request_memory, request_cpu, limit_memory, limit_cpu), default None + :param resources: dict: Specify cpu and memory resource usage (dict: request/limit: {"memory": "", "cpu": "", "ephemeral-storage": ""}), default None :param startup_timeout_seconds: int: pod startup timeout :param retries: int: Number of retries for task before DAG fails, default 3 :param delete_on_finish: bool: Whether to delete pod on completion diff --git a/dataverk_airflow/notebook_operator.py b/dataverk_airflow/notebook_operator.py index 0725b24..dc9827a 100644 --- a/dataverk_airflow/notebook_operator.py +++ b/dataverk_airflow/notebook_operator.py @@ -44,7 +44,7 @@ def notebook_operator( :param extra_envs: dict: dict with environment variables example: {"key": "value", "key2": "value2"} :param allowlist: list: list of hosts and port the task needs to reach on the format host:port :param requirements_path: bool: Path (including filename) to your requirements.txt - :param resources: dict: Specify required cpu and memory requirements (keys in dict: request_memory, request_cpu, limit_memory, limit_cpu), default None + :param resources: dict: Specify cpu and memory resource usage (dict: request/limit: {"memory": "", "cpu": "", "ephemeral-storage": ""}), default None :param startup_timeout_seconds: int: pod startup timeout :param retries: int: Number of retries for task before DAG fails, default 3 :param delete_on_finish: bool: Whether to delete pod on completion diff --git a/dataverk_airflow/quarto_operator.py b/dataverk_airflow/quarto_operator.py index f215f9f..cb5642f 100644 --- a/dataverk_airflow/quarto_operator.py +++ b/dataverk_airflow/quarto_operator.py @@ -42,7 +42,7 @@ def quarto_operator( :param extra_envs: dict: Dict with environment variables, example: {"key": "value", "key2": "value2"} :param allowlist: list: List of hosts and port the task needs to reach on the format host:port :param requirements_path: bool: Path (including filename) to your requirements.txt - :param resources: dict: Specify required cpu and memory requirements (keys in dict: request_memory, request_cpu, limit_memory, limit_cpu), default None + :param resources: dict: Specify cpu and memory resource usage (dict: request/limit: {"memory": "", "cpu": "", "ephemeral-storage": ""}), default None :param startup_timeout_seconds: int: Pod startup timeout :param retries: int: Number of retries for task before DAG fails, default 3 :param delete_on_finish: bool: Whether to delete pod on completion